Fix issues in text selection

  - PR #13257 fixed a lot of issues but not all and this patch aims to fix almost all remaining issues.
  - the idea in this new patch is to compare position of new glyph with the last position where a glyph has been drawn;
    - no space are "drawn": it just moves the cursor but they aren't added in the chunk;
    - so this way a space followed by a cursor move can be treated as only one space: it helps to merge all spaces into one.
  - to make difference between real spaces and tracking ones, we used a factor of the space width (from the font)
    - it was a pretty good idea in general but it fails with some fonts where space was too big:
    - in Poppler, they're using a factor of the font size: this is an excellent idea (<= 0.1 * fontSize implies tracking space).

src/core/fonts.js

@@ -2884,7 +2884,7 @@ class Font {
   /**
    * @private
    */
-  _charToGlyph(charcode, isSpace = false) {
+  _charToGlyph(charcode) {
     let fontCharCode, width, operatorListId;
 
     let widthCode = charcode;
@@ -2903,6 +2903,7 @@ class Font {
       unicode = String.fromCharCode(unicode);
     }
 
+    const isSpace = unicode === " ";
     let isInFont = charcode in this.toFontChar;
     // First try the toFontChar map, if it's not there then try falling
     // back to the char code.
@@ -3006,15 +3007,13 @@ class Font {
         charcode = c.charcode;
         const length = c.length;
         i += length;
-        // Space is char with code 0x20 and length 1 in multiple-byte codes.
-        const isSpace = length === 1 && chars.charCodeAt(i - 1) === 0x20;
-        glyph = this._charToGlyph(charcode, isSpace);
+        glyph = this._charToGlyph(charcode);
         glyphs.push(glyph);
       }
     } else {
       for (i = 0, ii = chars.length; i < ii; ++i) {
         charcode = chars.charCodeAt(i);
-        glyph = this._charToGlyph(charcode, charcode === 0x20);
+        glyph = this._charToGlyph(charcode);
         glyphs.push(glyph);
       }
     }

test/pdfs/.gitignore

@@ -12,6 +12,7 @@
 !xref_command_missing.pdf
 !issue1155r.pdf
 !issue2017r.pdf
+!issue11913.pdf
 !issue2391-1.pdf
 !issue2391-2.pdf
 !issue3214.pdf
@@ -171,6 +172,7 @@
 !issue11931.pdf
 !issue1655r.pdf
 !issue6541.pdf
+!issue10640.pdf
 !issue2948.pdf
 !issue6231_1.pdf
 !issue10402.pdf
@@ -269,6 +271,7 @@
 !issue2840.pdf
 !issue4061.pdf
 !issue4668.pdf
+!issue13226.pdf
 !PDFJS-7562-reduced.pdf
 !issue11768_reduced.pdf
 !issue5039.pdf
@@ -420,6 +423,7 @@
 !annotation-fileattachment.pdf
 !annotation-text-widget.pdf
 !annotation-choice-widget.pdf
+!issue10900.pdf
 !annotation-button-widget.pdf
 !annotation-polyline-polygon.pdf
 !annotation-polyline-polygon-without-appearance.pdf
@@ -440,6 +444,7 @@
 !issue9972-3.pdf
 !tiling-pattern-box.pdf
 !tiling-pattern-large-steps.pdf
+!issue13201.pdf
 !issue11555.pdf
 !issue12337.pdf
 !pr12564.pdf

test/test_manifest.json

@@ -97,6 +97,12 @@
        "link": false,
        "type": "text"
     },
+    {  "id": "issue10900",
+       "file": "pdfs/issue10900.pdf",
+       "md5": "78e598c748076a1dc9e67bbec1a1b666",
+       "rounds": 1,
+       "type": "other"
+    },
     {  "id": "issue1293",
        "file": "pdfs/issue1293r.pdf",
        "md5": "4a098f5051f34fab036f5bbe88f8deef",
@@ -1282,6 +1288,12 @@
        "rounds": 1,
        "type": "text"
     },
+    {  "id": "issue13201",
+       "file": "pdfs/issue13201.pdf",
+       "md5": "697ce2a55d39c5ad154f2a6ec9ead848",
+       "rounds": 1,
+       "type": "other"
+    },
     {  "id": "bug1250079",
        "file": "pdfs/bug1250079.pdf",
        "md5": "a1dd21a70ae7097d96273e85a80b26ef",
@@ -3186,6 +3198,12 @@
       "link": true,
       "type": "eq"
     },
+    {  "id": "issue11913",
+       "file": "pdfs/issue11913.pdf",
+       "md5": "d2aabb48440b297b02a2e8405d330f65",
+       "rounds": 1,
+       "type": "other"
+    },
     {  "id": "issue6117",
        "file": "pdfs/issue6117.pdf",
        "md5": "691f5f8268e07f3831e8293258a68da7",
@@ -4421,6 +4439,12 @@
          }
        }
     },
+    {  "id": "issue13226",
+       "file": "pdfs/issue13226.pdf",
+       "md5": "080eac5a045b7759cb916604f88c7620",
+       "rounds": 1,
+       "type": "other"
+    },
     {  "id": "issue1171.pdf",
        "file": "pdfs/issue1171.pdf",
        "md5": "2a6188a42a5874c7874b88eebd4acaf0",
@@ -4537,6 +4561,12 @@
        "rounds": 1,
        "type": "eq"
     },
+    {  "id": "issue10640",
+       "file": "pdfs/issue10640.pdf",
+       "md5": "7da503dec900d4dc3996fd9be4457c2f",
+       "rounds": 1,
+       "type": "other"
+    },
     {  "id": "issue12392",
        "file": "pdfs/issue12392.pdf",
        "md5": "76c3a34c6520940c45c66c92f7df2de5",

test/unit/api_spec.js

@@ -70,6 +70,10 @@ describe("api", function () {
     }, WAIT_TIMEOUT);
   }
 
+  function mergeText(items) {
+    return items.map(chunk => chunk.str + (chunk.hasEOL ? "\n" : "")).join("");
+  }
+
   describe("getDocument", function () {
     it("creates pdf doc from URL-string", async function () {
       const urlStr = TEST_PDFS_PATH + basicApiFileName;
@@ -1500,13 +1504,20 @@ describe("api", function () {
       });
 
       const data = await Promise.all([defaultPromise, parametersPromise]);
+      // console.log(data);window.alert("");
 
       expect(!!data[0].items).toEqual(true);
-      expect(data[0].items.length).toEqual(12);
+      expect(data[0].items.length).toEqual(11);
       expect(!!data[0].styles).toEqual(true);
 
+      const page1 = mergeText(data[0].items);
+      expect(page1).toEqual(`Table Of Content
+Chapter 1 .......................................................... 2
+Paragraph 1.1 ...................................................... 3
+page 1 / 3`);
+
       expect(!!data[1].items).toEqual(true);
-      expect(data[1].items.length).toEqual(7);
+      expect(data[1].items.length).toEqual(6);
       expect(!!data[1].styles).toEqual(true);
     });
 
@@ -1539,6 +1550,107 @@ describe("api", function () {
       await loadingTask.destroy();
     });
 
+    it("gets text content, with no extra spaces (issue 13226)", async function () {
+      const loadingTask = getDocument(buildGetDocumentParams("issue13226.pdf"));
+      const pdfDoc = await loadingTask.promise;
+      const pdfPage = await pdfDoc.getPage(1);
+      const { items } = await pdfPage.getTextContent();
+      const text = mergeText(items);
+
+      expect(text).toEqual(
+        "Mitarbeiterinnen und Mitarbeiter arbeiten in über 100 Ländern engagiert im Dienste"
+      );
+
+      await loadingTask.destroy();
+    });
+
+    it("gets text content, with merged spaces (issue 13201)", async function () {
+      const loadingTask = getDocument(buildGetDocumentParams("issue13201.pdf"));
+      const pdfDoc = await loadingTask.promise;
+      const pdfPage = await pdfDoc.getPage(1);
+      const { items } = await pdfPage.getTextContent();
+      const text = mergeText(items);
+
+      expect(
+        text.includes(
+          "Abstract. A purely peer-to-peer version of electronic cash would allow online"
+        )
+      ).toEqual(true);
+      expect(
+        text.includes(
+          "avoid mediating disputes. The cost of mediation increases transaction costs, limiting the"
+        )
+      ).toEqual(true);
+      expect(
+        text.includes(
+          "system is secure as long as honest nodes collectively control more CPU power than any"
+        )
+      ).toEqual(true);
+
+      await loadingTask.destroy();
+    });
+
+    it("gets text content, with no spaces between letters of words (issue 11913)", async function () {
+      const loadingTask = getDocument(buildGetDocumentParams("issue11913.pdf"));
+      const pdfDoc = await loadingTask.promise;
+      const pdfPage = await pdfDoc.getPage(1);
+      const { items } = await pdfPage.getTextContent();
+      const text = mergeText(items);
+
+      expect(
+        text.includes(
+          "1. The first of these cases arises from the tragic handicap which has blighted the life of the Plaintiff, and from the response of the"
+        )
+      ).toEqual(true);
+      expect(
+        text.includes(
+          "argued in this Court the appeal raises narrower, but important, issues which may be summarised as follows:-"
+        )
+      ).toEqual(true);
+      await loadingTask.destroy();
+    });
+
+    it("gets text content, with merged (issue 10900)", async function () {
+      const loadingTask = getDocument(buildGetDocumentParams("issue10900.pdf"));
+      const pdfDoc = await loadingTask.promise;
+      const pdfPage = await pdfDoc.getPage(1);
+      const { items } = await pdfPage.getTextContent();
+      const text = mergeText(items);
+
+      expect(
+        text.includes(`3 3 3 3
+851.5 854.9 839.3 837.5
+633.6 727.8 789.9 796.2
+1,485.1 1,582.7 1,629.2 1,633.7
+114.2 121.7 125.3 130.7
+13.0x 13.0x 13.0x 12.5x`)
+      ).toEqual(true);
+
+      await loadingTask.destroy();
+    });
+
+    it("gets text content, with spaces", async function () {
+      const loadingTask = getDocument(buildGetDocumentParams("issue10640.pdf"));
+      const pdfDoc = await loadingTask.promise;
+      const pdfPage = await pdfDoc.getPage(1);
+      const { items } = await pdfPage.getTextContent();
+      const text = mergeText(items);
+
+      expect(
+        text.includes(`Open Sans is a humanist sans serif typeface designed by Steve Matteson.
+Open Sans was designed with an upright stress, open forms and a neu-
+tral, yet friendly appearance. It was optimized for print, web, and mobile
+interfaces, and has excellent legibility characteristics in its letterforms (see
+figure \x81 on the following page). This font is available from the Google Font
+Directory [\x81] as TrueType files licensed under the Apache License version \x82.\x80.
+This package provides support for this font in LATEX. It includes Type \x81
+versions of the fonts, converted for this package using FontForge from its
+sources, for full support with Dvips.`)
+      ).toEqual(true);
+
+      await loadingTask.destroy();
+    });
+
     it("gets empty structure tree", async function () {
       const tree = await page.getStructTree();
 

test/unit/pdf_find_controller_spec.js

@@ -268,7 +268,7 @@ describe("pdf_find_controller", function () {
         pageIndex: 0,
         matchIndex: 0,
       },
-      pageMatches: [[19, 48, 66]],
+      pageMatches: [[19, 46, 62]],
       pageMatchesLength: [[8, 8, 8]],
     });
   });