feat: allow disabling tokenizers

__tests__/__snapshots__/disabling-tokenizers.test.js.snap

@@ -0,0 +1,69 @@
+// Jest Snapshot v1, https://goo.gl/fbAQLP
+
+exports[`disableTokenizers: "blocks" disabling block tokenizer 1`] = `
+Object {
+  "children": Array [
+    Object {
+      "children": Array [
+        Object {
+          "type": "text",
+          "value": "#  heading 1",
+        },
+      ],
+      "type": "paragraph",
+    },
+  ],
+  "type": "root",
+}
+`;
+
+exports[`disableTokenizers: "inlines" disabling delete 1`] = `
+Object {
+  "children": Array [
+    Object {
+      "children": Array [
+        Object {
+          "type": "text",
+          "value": "~~strikethrough~~",
+        },
+      ],
+      "type": "paragraph",
+    },
+  ],
+  "type": "root",
+}
+`;
+
+exports[`disableTokenizers: "inlines" disabling emphasis 1`] = `
+Object {
+  "children": Array [
+    Object {
+      "children": Array [
+        Object {
+          "type": "text",
+          "value": "*emphatic **strong** emphatic*",
+        },
+      ],
+      "type": "paragraph",
+    },
+  ],
+  "type": "root",
+}
+`;
+
+exports[`disableTokenizers: "inlines" disabling inlineCode 1`] = `
+Object {
+  "children": Array [
+    Object {
+      "children": Array [
+        Object {
+          "type": "text",
+          "value": "\`const js = true \`",
+        },
+      ],
+      "type": "paragraph",
+    },
+  ],
+  "type": "root",
+}
+`;

__tests__/__snapshots__/link-parsers.test.js.snap

@@ -0,0 +1,122 @@
+// Jest Snapshot v1, https://goo.gl/fbAQLP
+
+exports[`a bare autoLinked url 1`] = `
+Object {
+  "children": Array [
+    Object {
+      "children": Array [
+        Object {
+          "children": Array [
+            Object {
+              "type": "text",
+              "value": "http://www.googl.com",
+            },
+          ],
+          "title": null,
+          "type": "link",
+          "url": "http://www.googl.com",
+        },
+      ],
+      "type": "paragraph",
+    },
+  ],
+  "type": "root",
+}
+`;
+
+exports[`a bare autoLinked url with no protocol 1`] = `
+Object {
+  "children": Array [
+    Object {
+      "children": Array [
+        Object {
+          "children": Array [
+            Object {
+              "type": "text",
+              "value": "www.google.com",
+            },
+          ],
+          "title": null,
+          "type": "link",
+          "url": "http://www.google.com",
+        },
+      ],
+      "type": "paragraph",
+    },
+  ],
+  "type": "root",
+}
+`;
+
+exports[`a bracketed autoLinked url 1`] = `
+Object {
+  "children": Array [
+    Object {
+      "children": Array [
+        Object {
+          "children": Array [
+            Object {
+              "type": "text",
+              "value": "http://www.google.com",
+            },
+          ],
+          "title": null,
+          "type": "link",
+          "url": "http://www.google.com",
+        },
+      ],
+      "type": "paragraph",
+    },
+  ],
+  "type": "root",
+}
+`;
+
+exports[`a link ref 1`] = `
+Object {
+  "children": Array [
+    Object {
+      "children": Array [
+        Object {
+          "children": Array [
+            Object {
+              "type": "text",
+              "value": "link",
+            },
+          ],
+          "identifier": "link",
+          "label": "link",
+          "referenceType": "shortcut",
+          "type": "linkReference",
+        },
+      ],
+      "type": "paragraph",
+    },
+  ],
+  "type": "root",
+}
+`;
+
+exports[`a link with label 1`] = `
+Object {
+  "children": Array [
+    Object {
+      "children": Array [
+        Object {
+          "children": Array [
+            Object {
+              "type": "text",
+              "value": "link",
+            },
+          ],
+          "title": null,
+          "type": "link",
+          "url": "http://www.foo.com",
+        },
+      ],
+      "type": "paragraph",
+    },
+  ],
+  "type": "root",
+}
+`;

__tests__/disabling-tokenizers.test.js

@@ -0,0 +1,29 @@
+const markdown = require('../index');
+
+describe('disableTokenizers: "inlines"', () => {
+  const opts = { disableTokenizers: 'inlines' };
+
+  it('disabling inlineCode', () => {
+    const md = '`const js = true `';
+    expect(markdown.mdast(md, opts)).toMatchSnapshot();
+  });
+
+  it('disabling emphasis', () => {
+    const md = '*emphatic **strong** emphatic*';
+    expect(markdown.mdast(md, opts)).toMatchSnapshot();
+  });
+
+  it('disabling delete', () => {
+    const md = '~~strikethrough~~';
+    expect(markdown.mdast(md, opts)).toMatchSnapshot();
+  });
+});
+
+describe('disableTokenizers: "blocks"', () => {
+  const opts = { disableTokenizers: 'blocks' };
+
+  it('disabling block tokenizer', () => {
+    const md = '# heading 1';
+    expect(markdown.mdast(md, opts)).toMatchSnapshot();
+  });
+});

__tests__/flavored-compilers.test.js

@@ -7,7 +7,7 @@ const rehypeSanitize = require('rehype-sanitize');
 
 const parsers = Object.values(require('../processor/parse')).map(parser => parser.sanitize(sanitize));
 const compilers = Object.values(require('../processor/compile'));
-const options = require('../options.json').markdownOptions;
+const options = require('../options.js').options.markdownOptions;
 
 const processor = unified()
   .use(remarkParse, options)

__tests__/flavored-parsers.test.js

@@ -4,7 +4,7 @@ const rehypeSanitize = require('rehype-sanitize');
 
 const parseCallouts = require('../processor/parse/flavored/callout');
 const parseCodeTabs = require('../processor/parse/flavored/code-tabs');
-const options = require('../options.json').markdownOptions;
+const options = require('../options.js').options.markdownOptions;
 
 const sanitize = { attributes: [], tagNames: [] };
 const process = (text, opts = options) =>

__tests__/index.test.js

@@ -4,7 +4,7 @@ const BaseUrlContext = require('../contexts/BaseUrl');
 
 const markdown = require('../index');
 const { tableFlattening } = require('../processor/plugin/table-flattening');
-const settings = require('../options.json');
+const { options } = require('../options.js');
 
 test('image', () => {
   expect(mount(markdown.default('![Image](http://example.com/image.png)')).html()).toMatchSnapshot();
@@ -33,7 +33,7 @@ test('magic image', () => {
     }
     [/block]
     `,
-        settings
+        options
       )
     ).html()
   ).toMatchSnapshot();

__tests__/link-parsers.test.js

@@ -0,0 +1,21 @@
+const markdown = require('../index');
+
+test('a link with label', () => {
+  expect(markdown.mdast('[link](http://www.foo.com)')).toMatchSnapshot();
+});
+
+test('a link ref', () => {
+  expect(markdown.mdast('[link]')).toMatchSnapshot();
+});
+
+test('a bracketed autoLinked url', () => {
+  expect(markdown.mdast('<http://www.google.com>')).toMatchSnapshot();
+});
+
+test('a bare autoLinked url', () => {
+  expect(markdown.mdast('http://www.googl.com')).toMatchSnapshot();
+});
+
+test.skip('a bare autoLinked url with no protocol', () => {
+  expect(markdown.mdast('www.google.com')).toMatchSnapshot();
+});

__tests__/magic-block-parser.test.js

@@ -3,7 +3,7 @@ const remarkParse = require('remark-parse');
 const rehypeSanitize = require('rehype-sanitize');
 
 const parser = require('../processor/parse/magic-block-parser');
-const options = require('../options.json').markdownOptions;
+const options = require('../options.js').options.markdownOptions;
 
 const { silenceConsole } = require('./helpers');
 

__tests__/magic-block-parser/table.test.js

@@ -3,7 +3,7 @@ const remarkParse = require('remark-parse');
 const rehypeSanitize = require('rehype-sanitize');
 
 const parser = require('../../processor/parse/magic-block-parser');
-const options = require('../../options.json').markdownOptions;
+const options = require('../../options.js').options.markdownOptions;
 
 const sanitize = { attributes: [] };
 const process = (text, opts = options) =>

docs/getting-started.md

@@ -66,6 +66,7 @@ The `utils` export gives you access to various tools and configuration settings:
   - `markdownOptions`—configuration object passed to `remark`
   - `correctnewlines`—flag to toggle newline transformation.
   - `normalize`—auto-normalize magic blocks before processing.
+  - `disableTokenizers`—disable internal `block` or `inline` tokenizers.
 - **`<GlossaryContext/>`** and **`<VariablesContext/>`**
   React provider and consumer wrappers for [user data injection](doc:features#section-data-injection).
 [block:html]

index.js

@@ -21,6 +21,7 @@ const remarkStringify = require('remark-stringify');
 const remarkBreaks = require('remark-breaks');
 const remarkSlug = require('remark-slug');
 const remarkFrontmatter = require('remark-frontmatter');
+const remarkDisableTokenizers = require('remark-disable-tokenizers');
 
 // rehype plugins
 const rehypeSanitize = require('rehype-sanitize');
@@ -61,14 +62,14 @@ const tableFlattening = require('./processor/plugin/table-flattening');
 const toPlainText = require('./processor/plugin/plain-text');
 
 // Processor Option Defaults
-const options = require('./options.json');
+const { options, parseOptions } = require('./options.js');
 
 /**
  * Normalize Magic Block Raw Text
  */
 export function setup(blocks, opts = {}) {
   // merge default and user options
-  opts = { ...options, ...opts };
+  opts = parseOptions(opts);
 
   // normalize magic block linebreaks
   if (opts.normalize && blocks) {
@@ -110,6 +111,7 @@ export function processor(opts = {}) {
    * - sanitize and remove any disallowed attributes
    * - output the hast to a React vdom with our custom components
    */
+
   return unified()
     .use(remarkParse, opts.markdownOptions)
     .use(remarkFrontmatter, ['yaml', 'toml'])
@@ -118,6 +120,7 @@ export function processor(opts = {}) {
     .use(!opts.correctnewlines ? remarkBreaks : () => {})
     .use(customParsers)
     .use(remarkSlug)
+    .use(remarkDisableTokenizers, opts.disableTokenizers)
     .use(remarkRehype, { allowDangerousHtml: true })
     .use(rehypeRaw)
     .use(rehypeSanitize, sanitize);