feat: allow disabling tokenizers

__tests__/__snapshots__/disabling-tokenizers.test.js.snap

@@ -0,0 +1,69 @@
+// Jest Snapshot v1, https://goo.gl/fbAQLP
+
+exports[`disabling block tokenizer 1`] = `
+Object {
+  "children": Array [
+    Object {
+      "children": Array [
+        Object {
+          "type": "text",
+          "value": "#  heading 1",
+        },
+      ],
+      "type": "paragraph",
+    },
+  ],
+  "type": "root",
+}
+`;
+
+exports[`disabling delete 1`] = `
+Object {
+  "children": Array [
+    Object {
+      "children": Array [
+        Object {
+          "type": "text",
+          "value": "~~strikethrough~~",
+        },
+      ],
+      "type": "paragraph",
+    },
+  ],
+  "type": "root",
+}
+`;
+
+exports[`disabling emphasis 1`] = `
+Object {
+  "children": Array [
+    Object {
+      "children": Array [
+        Object {
+          "type": "text",
+          "value": "*emphatic **strong** emphatic*",
+        },
+      ],
+      "type": "paragraph",
+    },
+  ],
+  "type": "root",
+}
+`;
+
+exports[`disabling inlineCode 1`] = `
+Object {
+  "children": Array [
+    Object {
+      "children": Array [
+        Object {
+          "type": "text",
+          "value": "\`const js = true \`",
+        },
+      ],
+      "type": "paragraph",
+    },
+  ],
+  "type": "root",
+}
+`;

__tests__/__snapshots__/link-parsers.test.js.snap

@@ -0,0 +1,115 @@
+// Jest Snapshot v1, https://goo.gl/fbAQLP
+
+exports[`a bare autoLinked url 1`] = `
+Object {
+  "children": Array [
+    Object {
+      "children": Array [
+        Object {
+          "children": Array [
+            Object {
+              "type": "text",
+              "value": "http://www.google.com",
+            },
+          ],
+          "title": null,
+          "type": "link",
+          "url": "http://www.google.com",
+        },
+      ],
+      "type": "paragraph",
+    },
+  ],
+  "type": "root",
+}
+`;
+
+exports[`a bare autoLinked url with no protocol 1`] = `
+Object {
+  "children": Array [
+    Object {
+      "children": Array [
+        Object {
+          "type": "text",
+          "value": "www.google.com",
+        },
+      ],
+      "type": "paragraph",
+    },
+  ],
+  "type": "root",
+}
+`;
+
+exports[`a bracketed autoLinked url 1`] = `
+Object {
+  "children": Array [
+    Object {
+      "children": Array [
+        Object {
+          "children": Array [
+            Object {
+              "type": "text",
+              "value": "http://www.google.com",
+            },
+          ],
+          "title": null,
+          "type": "link",
+          "url": "http://www.google.com",
+        },
+      ],
+      "type": "paragraph",
+    },
+  ],
+  "type": "root",
+}
+`;
+
+exports[`a link ref 1`] = `
+Object {
+  "children": Array [
+    Object {
+      "children": Array [
+        Object {
+          "children": Array [
+            Object {
+              "type": "text",
+              "value": "link",
+            },
+          ],
+          "identifier": "link",
+          "label": "link",
+          "referenceType": "shortcut",
+          "type": "linkReference",
+        },
+      ],
+      "type": "paragraph",
+    },
+  ],
+  "type": "root",
+}
+`;
+
+exports[`a link with label 1`] = `
+Object {
+  "children": Array [
+    Object {
+      "children": Array [
+        Object {
+          "children": Array [
+            Object {
+              "type": "text",
+              "value": "link",
+            },
+          ],
+          "title": null,
+          "type": "link",
+          "url": "http://www.foo.com",
+        },
+      ],
+      "type": "paragraph",
+    },
+  ],
+  "type": "root",
+}
+`;

__tests__/disabling-tokenizers.test.js

@@ -0,0 +1,29 @@
+const markdown = require('../index');
+
+test('disabling inlineCode', () => {
+  const md = '`const js = true `';
+  const opts = { disableTokenizers: { inline: ['code'] } };
+
+  expect(markdown.mdast(md, opts)).toMatchSnapshot();
+});
+
+test('disabling emphasis', () => {
+  const md = '*emphatic **strong** emphatic*';
+  const opts = { disableTokenizers: { inline: ['emphasis', 'strong'] } };
+
+  expect(markdown.mdast(md, opts)).toMatchSnapshot();
+});
+
+test('disabling delete', () => {
+  const md = '~~strikethrough~~';
+  const opts = { disableTokenizers: { inline: ['deletion'] } };
+
+  expect(markdown.mdast(md, opts)).toMatchSnapshot();
+});
+
+test('disabling block tokenizer', () => {
+  const md = '# heading 1';
+  const opts = { disableTokenizers: { block: ['atxHeading'] } };
+
+  expect(markdown.mdast(md, opts)).toMatchSnapshot();
+});

__tests__/link-parsers.test.js

@@ -0,0 +1,21 @@
+const markdown = require('../index');
+
+test('a link with label', () => {
+  expect(markdown.mdast('[link](http://www.foo.com)')).toMatchSnapshot();
+});
+
+test('a link ref', () => {
+  expect(markdown.mdast('[link]')).toMatchSnapshot();
+});
+
+test('a bracketed autoLinked url', () => {
+  expect(markdown.mdast('<http://www.google.com>')).toMatchSnapshot();
+});
+
+test('a bare autoLinked url', () => {
+  expect(markdown.mdast('http://www.googl.com')).toMatchSnapshot();
+});
+
+test('a bare autoLinked url with no protocol', () => {
+  expect(markdown.mdast('www.google.com')).toMatchSnapshot();
+});

index.js

@@ -21,6 +21,7 @@ const remarkStringify = require('remark-stringify');
 const remarkBreaks = require('remark-breaks');
 const remarkSlug = require('remark-slug');
 const remarkFrontmatter = require('remark-frontmatter');
+const remarkDisableTokenizers = require('remark-disable-tokenizers');
 
 // rehype plugins
 const rehypeSanitize = require('rehype-sanitize');
@@ -61,14 +62,14 @@ const tableFlattening = require('./processor/plugin/table-flattening');
 const toPlainText = require('./processor/plugin/plain-text');
 
 // Processor Option Defaults
-const options = require('./options.json');
+const { options, parseOptions } = require('./options.js');
 
 /**
  * Normalize Magic Block Raw Text
  */
 export function setup(blocks, opts = {}) {
   // merge default and user options
-  opts = { ...options, ...opts };
+  opts = parseOptions(opts);
 
   // normalize magic block linebreaks
   if (opts.normalize && blocks) {
@@ -110,6 +111,7 @@ export function processor(opts = {}) {
    * - sanitize and remove any disallowed attributes
    * - output the hast to a React vdom with our custom components
    */
+
   return unified()
     .use(remarkParse, opts.markdownOptions)
     .use(remarkFrontmatter, ['yaml', 'toml'])
@@ -118,6 +120,7 @@ export function processor(opts = {}) {
     .use(!opts.correctnewlines ? remarkBreaks : () => {})
     .use(customParsers)
     .use(remarkSlug)
+    .use(remarkDisableTokenizers, opts.disableTokenizers)
     .use(remarkRehype, { allowDangerousHtml: true })
     .use(rehypeRaw)
     .use(rehypeSanitize, sanitize);

options.js

@@ -0,0 +1,83 @@
+const options = {
+  compatibilityMode: false,
+  copyButtons: true,
+  correctnewlines: false,
+  markdownOptions: {
+    fences: true,
+    commonmark: true,
+    gfm: true,
+    ruleSpaces: false,
+    listItemIndent: '1',
+    spacedTable: true,
+    paddedTable: true,
+    setext: true,
+  },
+  normalize: true,
+  settings: {
+    position: false,
+  },
+};
+
+// NOTE: disabling newline, paragraph, or text trips remark into an infinite loop!
+const blocks = [
+  // 'newline',
+  'indentedCode',
+  'fencedCode',
+  'blockquote',
+  'atxHeading',
+  'thematicBreak',
+  'list',
+  'setextHeading',
+  'html',
+  'footnote',
+  'definition',
+  'table',
+  // 'paragraph',
+];
+
+const inlines = [
+  'escape',
+  'autoLink',
+  'url',
+  'html',
+  'link',
+  'reference',
+  'strong',
+  'emphasis',
+  'deletion',
+  'code',
+  'break',
+  // 'text',
+];
+
+const toBeDecorated = {
+  inlines: inlines.filter(i => !['link', 'reference'].includes(i)),
+  blocks: [],
+};
+
+const disableTokenizers = {
+  blocks: {
+    disableTokenizers: {
+      inline: toBeDecorated.inlines,
+      block: toBeDecorated.blocks,
+    },
+  },
+  inlines: {
+    disableTokenizers: {
+      inline: inlines.filter(i => !toBeDecorated.inlines.includes(i)),
+      block: blocks.filter(b => !toBeDecorated.blocks.includes(b)),
+    },
+  },
+};
+
+const parseOptions = (opts = {}) => {
+  if (opts.tokenizerSet in disableTokenizers) {
+    return { ...disableTokenizers[opts.tokenizerSet], ...opts };
+  } else if (opts.tokenizerSet) {
+    throw new Error(`opts.tokenizerSet "${opts.tokenizerSet}" not one of "${Object.keys(disableTokenizers)}"`);
+  }
+
+  return opts;
+};
+
+export { options, parseOptions };