Update to latest CodeMirror

src/editor/Editor.js

@@ -675,21 +675,23 @@ define(function (require, exports, module) {
     };
 
     /**
-     * Sets the contents of the editor and clears the undo/redo history. Dispatches a change event.
+     * Sets the contents of the editor, clears the undo/redo history and marks the document clean. Dispatches a change event.
      * Semi-private: only Document should call this.
      * @param {!string} text
      */
     Editor.prototype._resetText = function (text) {
-        var perfTimerName = PerfUtils.markStart("Edtitor._resetText()\t" + (!this.document || this.document.file.fullPath));
+        var perfTimerName = PerfUtils.markStart("Editor._resetText()\t" + (!this.document || this.document.file.fullPath));
 
         var cursorPos = this.getCursorPos(),
             scrollPos = this.getScrollPos();
 
         // This *will* fire a change event, but we clear the undo immediately afterward
         this._codeMirror.setValue(text);
 
-        // Make sure we can't undo back to the empty state before setValue()
+        // Make sure we can't undo back to the empty state before setValue(), and mark
+        // the document clean.
         this._codeMirror.clearHistory();
+        this._codeMirror.markClean();
 
         // restore cursor and scroll positions
         this.setCursorPos(cursorPos);

src/extensions/default/JavaScriptCodeHints/Session.js

@@ -119,9 +119,9 @@ define(function (require, exports, module) {
         var cm = this.editor._codeMirror;
 
         if (cursor) {
-            return cm.getTokenAt(cursor);
+            return cm.getTokenAt(cursor, true);
         } else {
-            return cm.getTokenAt(this.getCursor());
+            return cm.getTokenAt(this.getCursor(), true);
         }
     };
 

src/extensions/default/JavaScriptQuickEdit/main.js

@@ -44,12 +44,12 @@ define(function (require, exports, module) {
      * @return {String} token string at the specified position
      */
     function _getFunctionName(hostEditor, pos) {
-        var token = hostEditor._codeMirror.getTokenAt(pos);
+        var token = hostEditor._codeMirror.getTokenAt(pos, true);
 
         // If the pos is at the beginning of a name, token will be the 
         // preceding whitespace or dot. In that case, try the next pos.
         if (token.string.trim().length === 0 || token.string === ".") {
-            token = hostEditor._codeMirror.getTokenAt({line: pos.line, ch: pos.ch + 1});
+            token = hostEditor._codeMirror.getTokenAt({line: pos.line, ch: pos.ch + 1}, true);
         }
 
         // Return valid function expressions only (function call or reference)

src/extensions/default/QuickView/main.js

@@ -466,7 +466,7 @@ define(function (require, exports, module) {
             }
 
             // Query providers for a new popoverState
-            var token = cm.getTokenAt(pos);
+            var token = cm.getTokenAt(pos, true);
             popoverState = queryPreviewProviders(editor, pos, token);
 
             if (popoverState) {

src/extensions/default/QuickView/unittests.js

@@ -69,7 +69,7 @@ define(function (require, exports, module) {
                 token;
 
             editor.setCursorPos(pos);
-            token = cm.getTokenAt(pos);
+            token = cm.getTokenAt(pos, true);
 
             return QuickView._queryPreviewProviders(editor, pos, token);
         }

src/extensions/samples/InlineImageViewer/main.js

@@ -43,12 +43,12 @@ define(function (require, exports, module) {
      * @return {String} token string at the specified position
      */
     function _getStringAtPos(hostEditor, pos) {
-        var token = hostEditor._codeMirror.getTokenAt(pos);
+        var token = hostEditor._codeMirror.getTokenAt(pos, true);
 
         // If the pos is at the beginning of a name, token will be the 
         // preceding whitespace or dot. In that case, try the next pos.
         if (token.string.trim().length === 0 || token.string === ".") {
-            token = hostEditor._codeMirror.getTokenAt({line: pos.line, ch: pos.ch + 1});
+            token = hostEditor._codeMirror.getTokenAt({line: pos.line, ch: pos.ch + 1}, true);
         }
 
         if (token.type === "string") {

src/language/CSSUtils.js

@@ -292,7 +292,7 @@ define(function (require, exports, module) {
             offset = TokenUtils.offsetInToken(ctx),
             canAddNewOne = false,
             testPos = {ch: ctx.pos.ch + 1, line: ctx.pos.line},
-            testToken = editor._codeMirror.getTokenAt(testPos),
+            testToken = editor._codeMirror.getTokenAt(testPos, true),
             propName;
 
         // Get property name first. If we don't have a valid property name, then 
@@ -376,7 +376,7 @@ define(function (require, exports, module) {
             propValues = [],
             offset = TokenUtils.offsetInToken(ctx),
             testPos = {ch: ctx.pos.ch + 1, line: ctx.pos.line},
-            testToken = editor._codeMirror.getTokenAt(testPos);
+            testToken = editor._codeMirror.getTokenAt(testPos, true);
 
         // Currently only support url. May be null if starting to type
         if (ctx.token.className && ctx.token.className !== "string") {
@@ -454,7 +454,7 @@ define(function (require, exports, module) {
                 propName = ctx.token.string;
             } else {
                 var testPos = {ch: ctx.pos.ch + 1, line: ctx.pos.line},
-                    testToken = editor._codeMirror.getTokenAt(testPos);
+                    testToken = editor._codeMirror.getTokenAt(testPos, true);
 
                 if (testToken.type === "property" || testToken.type === "property error" || testToken.type === "tag") {
                     propName = testToken.string;

src/language/HTMLUtils.js

@@ -321,7 +321,7 @@ define(function (require, exports, module) {
             // EOL, it uses EOL, same token is returned, and the following condition fails,
             // so we don't need to worry about testPos being valid.
             var testPos = {ch: ctx.pos.ch + 1, line: ctx.pos.line},
-                testToken = editor._codeMirror.getTokenAt(testPos);
+                testToken = editor._codeMirror.getTokenAt(testPos, true);
 
             if (testToken.string.length > 0 && testToken.string.trim().length > 0 &&
                     testToken.string.charAt(0) !== ">") {

src/utils/TokenUtils.js

@@ -44,7 +44,7 @@ define(function (require, exports, module) {
         return {
             "editor": editor,
             "pos": pos,
-            "token": editor.getTokenAt(pos)
+            "token": editor.getTokenAt(pos, true)
         };
     }
 
@@ -64,7 +64,7 @@ define(function (require, exports, module) {
         } else {
             ctx.pos.ch = ctx.token.start;
         }
-        ctx.token = ctx.editor.getTokenAt(ctx.pos);
+        ctx.token = ctx.editor.getTokenAt(ctx.pos, true);
         return true;
     }
 
@@ -85,7 +85,7 @@ define(function (require, exports, module) {
         } else {
             ctx.pos.ch = ctx.token.end + 1;
         }
-        ctx.token = ctx.editor.getTokenAt(ctx.pos);
+        ctx.token = ctx.editor.getTokenAt(ctx.pos, true);
         return true;
     }
 
@@ -128,7 +128,7 @@ define(function (require, exports, module) {
      */
     function getModeAt(cm, pos) {
         var outerMode = cm.getMode(),
-            modeData = CodeMirror.innerMode(outerMode, cm.getTokenAt(pos).state),
+            modeData = CodeMirror.innerMode(outerMode, cm.getTokenAt(pos, true).state),
             name;
 
         name = (modeData.mode.name === "xml") ?

test/spec/EditorCommandHandlers-test.js

@@ -570,13 +570,6 @@ define(function (require, exports, module) {
             expect(myDocument.getText()).toEqual(expectedCommentedText);
             expectSel(expectedCommentedSel);
 
-            // Toggle comment off
-            // Can't immediately call BLOCK_COMMENT again to uncomment because CodeMirror might not
-            // be done re-tokenizing in response to the first toggle, and BLOCK_COMMENT depends on
-            // getting correct tokens. See #2335. Ideally we'd listen for onHighlightComplete() but
-            // it's not clear that will always get called (if CM decides no async work was needed).
-            // So we just wait until after the async tokenization must have been run.
-            waits(200);
             runs(function () {
                 CommandManager.execute(Commands.EDIT_BLOCK_COMMENT, myEditor);
                 expect(myDocument.getText()).toEqual(startingContent);