diff --git a/src/plots/cartesian/set_convert.js b/src/plots/cartesian/set_convert.js
index 55bb4bdcb98..d700c83be06 100644
--- a/src/plots/cartesian/set_convert.js
+++ b/src/plots/cartesian/set_convert.js
@@ -133,7 +133,7 @@ module.exports = function setConvert(ax, fullLayout) {
             if(ax._categoriesMap[v] !== undefined) {
                 return ax._categoriesMap[v];
             } else {
-                ax._categories.push(v);
+                ax._categories.push(typeof v === 'number' ? String(v) : v);
 
                 var curLength = ax._categories.length - 1;
                 ax._categoriesMap[v] = curLength;
diff --git a/src/traces/heatmap/calc.js b/src/traces/heatmap/calc.js
index 69633c278f3..5cd2243920f 100644
--- a/src/traces/heatmap/calc.js
+++ b/src/traces/heatmap/calc.js
@@ -62,8 +62,8 @@ module.exports = function calc(gd, trace) {
             y = trace._y;
             zIn = trace._z;
         } else {
-            x = trace.x ? xa.makeCalcdata(trace, 'x') : [];
-            y = trace.y ? ya.makeCalcdata(trace, 'y') : [];
+            x = trace._x = trace.x ? xa.makeCalcdata(trace, 'x') : [];
+            y = trace._y = trace.y ? ya.makeCalcdata(trace, 'y') : [];
         }
 
         x0 = trace.x0;
@@ -71,7 +71,7 @@ module.exports = function calc(gd, trace) {
         y0 = trace.y0;
         dy = trace.dy;
 
-        z = clean2dArray(zIn, trace.transpose);
+        z = clean2dArray(zIn, trace, xa, ya);
 
         if(isContour || trace.connectgaps) {
             trace._emptypoints = findEmpties(z);
diff --git a/src/traces/heatmap/clean_2d_array.js b/src/traces/heatmap/clean_2d_array.js
index 74d6fa84db4..13bc6432eeb 100644
--- a/src/traces/heatmap/clean_2d_array.js
+++ b/src/traces/heatmap/clean_2d_array.js
@@ -9,8 +9,10 @@
 'use strict';
 
 var isNumeric = require('fast-isnumeric');
+var Lib = require('../../lib');
+var BADNUM = require('../../constants/numerical').BADNUM;
 
-module.exports = function clean2dArray(zOld, transpose) {
+module.exports = function clean2dArray(zOld, trace, xa, ya) {
     var rowlen, collen, getCollen, old2new, i, j;
 
     function cleanZvalue(v) {
@@ -18,7 +20,7 @@ module.exports = function clean2dArray(zOld, transpose) {
         return +v;
     }
 
-    if(transpose) {
+    if(trace && trace.transpose) {
         rowlen = 0;
         for(i = 0; i < zOld.length; i++) rowlen = Math.max(rowlen, zOld[i].length);
         if(rowlen === 0) return false;
@@ -30,12 +32,43 @@ module.exports = function clean2dArray(zOld, transpose) {
         old2new = function(zOld, i, j) { return zOld[i][j]; };
     }
 
+    var padOld2new = function(zOld, i, j) {
+        if(i === BADNUM || j === BADNUM) return BADNUM;
+        return old2new(zOld, i, j);
+    };
+
+    function axisMapping(ax) {
+        if(trace && trace.type !== 'carpet' && trace.type !== 'contourcarpet' &&
+            ax && ax.type === 'category' && trace['_' + ax._id.charAt(0)].length) {
+            var axLetter = ax._id.charAt(0);
+            var axMapping = {};
+            var traceCategories = trace['_' + axLetter + 'CategoryMap'] || trace[axLetter];
+            for(i = 0; i < traceCategories.length; i++) {
+                axMapping[traceCategories[i]] = i;
+            }
+            return function(i) {
+                var ind = axMapping[ax._categories[i]];
+                return ind + 1 ? ind : BADNUM;
+            };
+        } else {
+            return Lib.identity;
+        }
+    }
+
+    var xMap = axisMapping(xa);
+    var yMap = axisMapping(ya);
+
     var zNew = new Array(rowlen);
 
+    if(ya && ya.type === 'category') rowlen = ya._categories.length;
     for(i = 0; i < rowlen; i++) {
-        collen = getCollen(zOld, i);
+        if(xa && xa.type === 'category') {
+            collen = xa._categories.length;
+        } else {
+            collen = getCollen(zOld, i);
+        }
         zNew[i] = new Array(collen);
-        for(j = 0; j < collen; j++) zNew[i][j] = cleanZvalue(old2new(zOld, i, j));
+        for(j = 0; j < collen; j++) zNew[i][j] = cleanZvalue(padOld2new(zOld, yMap(i), xMap(j)));
     }
 
     return zNew;
diff --git a/src/traces/heatmap/convert_column_xyz.js b/src/traces/heatmap/convert_column_xyz.js
index ace8224f877..6a5d73721bf 100644
--- a/src/traces/heatmap/convert_column_xyz.js
+++ b/src/traces/heatmap/convert_column_xyz.js
@@ -65,4 +65,12 @@ module.exports = function convertColumnData(trace, ax1, ax2, var1Name, var2Name,
     }
     if(hasColumnText) trace._text = text;
     if(hasColumnHoverText) trace._hovertext = hovertext;
+
+    if(ax1 && ax1.type === 'category') {
+        trace['_' + var1Name + 'CategoryMap'] = col1vals.map(function(v) { return ax1._categories[v];});
+    }
+
+    if(ax2 && ax2.type === 'category') {
+        trace['_' + var2Name + 'CategoryMap'] = col2vals.map(function(v) { return ax2._categories[v];});
+    }
 };
diff --git a/src/traces/heatmap/hover.js b/src/traces/heatmap/hover.js
index 55e9f63d195..7c0deec8e0a 100644
--- a/src/traces/heatmap/hover.js
+++ b/src/traces/heatmap/hover.js
@@ -78,6 +78,10 @@ module.exports = function hoverPoints(pointData, xval, yval, hovermode, hoverLay
     } else {
         xl = xc ? xc[nx] : ((x[nx] + x[nx + 1]) / 2);
         yl = yc ? yc[ny] : ((y[ny] + y[ny + 1]) / 2);
+
+        if(xa && xa.type === 'category') xl = x[nx];
+        if(ya && ya.type === 'category') yl = y[ny];
+
         if(trace.zsmooth) {
             x0 = x1 = xa.c2p(xl);
             y0 = y1 = ya.c2p(yl);
diff --git a/test/image/baselines/heatmap_categoryorder.png b/test/image/baselines/heatmap_categoryorder.png
new file mode 100644
index 00000000000..c4613c59db0
Binary files /dev/null and b/test/image/baselines/heatmap_categoryorder.png differ
diff --git a/test/image/baselines/heatmap_columnar.png b/test/image/baselines/heatmap_columnar.png
new file mode 100644
index 00000000000..65a278a36ee
Binary files /dev/null and b/test/image/baselines/heatmap_columnar.png differ
diff --git a/test/image/baselines/heatmap_shared_categories.png b/test/image/baselines/heatmap_shared_categories.png
new file mode 100644
index 00000000000..93d8fb7a4c6
Binary files /dev/null and b/test/image/baselines/heatmap_shared_categories.png differ
diff --git a/test/image/mocks/heatmap_categoryorder.json b/test/image/mocks/heatmap_categoryorder.json
new file mode 100644
index 00000000000..be1c521e398
--- /dev/null
+++ b/test/image/mocks/heatmap_categoryorder.json
@@ -0,0 +1,26 @@
+{
+  "layout": {
+    "xaxis": {
+      "type": "category",
+      "categoryorder": "category descending"
+    },
+    "yaxis": {
+      "type": "category",
+      "categoryorder": "category descending"
+    },
+    "height": 400,
+    "width": 400
+  },
+  "data": [{
+    "type": "heatmap",
+
+    "x": ["z", "y", "x", "w"],
+    "y": ["d", "c", "b", "a"],
+    "z": [
+      [100, 75, 50, 0],
+      [90, 65, 40, 0],
+      [80, 55, 30, 0],
+      [0, 0, 0, 0]
+    ]
+  }]
+}
diff --git a/test/image/mocks/heatmap_columnar.json b/test/image/mocks/heatmap_columnar.json
new file mode 100644
index 00000000000..8c75c673a0e
--- /dev/null
+++ b/test/image/mocks/heatmap_columnar.json
@@ -0,0 +1,13 @@
+{
+  "data": [{
+    "type": "heatmap",
+    "x": ["a", "a", "a", "b", "b", "b", "c", "c", "c"],
+    "y": ["A", "B", "C", "A", "B", "C", "A", "B", "C"],
+    "z": [0, 50, 100, 50, 0, 255, 100, 510, 1010]
+  }],
+  "layout": {
+      "xaxis": {
+        "categoryorder": "category descending"
+      }
+  }
+}
diff --git a/test/image/mocks/heatmap_shared_categories.json b/test/image/mocks/heatmap_shared_categories.json
new file mode 100644
index 00000000000..c33fa64e298
--- /dev/null
+++ b/test/image/mocks/heatmap_shared_categories.json
@@ -0,0 +1,54 @@
+{
+  "data": [{
+    "type": "heatmap",
+    "x": ["Team A", "Team B", "Team C"],
+    "xaxis": "x",
+    "y": ["Game Three", "Game Two", "Game One"],
+    "z": [
+      [0.1, 0.3, 0.5],
+      [1, 0.8, 0.6],
+      [0.6, 0.4, 0.2]
+    ],
+    "yaxis": "y",
+    "coloraxis": "coloraxis"
+  }, {
+    "type": "heatmap",
+    "x": ["Team B", "Team C"],
+    "xaxis": "x",
+    "y": ["Game Three", "Game Two", "Game One"],
+    "z": [
+      [0.3, 0.5],
+      [0.8, 0.6],
+      [0.4, 0.2]
+    ],
+    "yaxis": "y2",
+    "coloraxis": "coloraxis"
+  }],
+  "layout": {
+    "xaxis": {
+      "anchor": "y2",
+      "domain": [0, 1],
+      "type": "category",
+      "range": [-0.5, 2.5],
+      "autorange": true
+    },
+    "yaxis": {
+      "anchor": "free",
+      "domain": [0.575, 1],
+      "position": 0,
+      "type": "category",
+      "range": [-0.5, 2.5],
+      "autorange": true
+    },
+    "yaxis2": {
+      "anchor": "x",
+      "domain": [0, 0.425],
+      "type": "category",
+      "range": [-0.5, 2.5],
+      "autorange": true
+    },
+    "coloraxis": {
+        "colorscale": "RdBu"
+    }
+  }
+}
diff --git a/test/jasmine/tests/axes_test.js b/test/jasmine/tests/axes_test.js
index 4409fad89b5..7507d27ea9e 100644
--- a/test/jasmine/tests/axes_test.js
+++ b/test/jasmine/tests/axes_test.js
@@ -3096,7 +3096,7 @@ describe('Test axes', function() {
                     x: new Float32Array([3, 1, 2]),
                 }, 'x', 'category');
                 expect(out).toEqual([0, 1, 2]);
-                expect(ax._categories).toEqual([3, 1, 2]);
+                expect(ax._categories).toEqual(['3', '1', '2']);
             });
 
             it('- on a date axis', function() {
diff --git a/test/jasmine/tests/calcdata_test.js b/test/jasmine/tests/calcdata_test.js
index 0f6ae376d68..430bfc5e7ca 100644
--- a/test/jasmine/tests/calcdata_test.js
+++ b/test/jasmine/tests/calcdata_test.js
@@ -862,7 +862,7 @@ describe('calculated data and points', function() {
                 xaxis: {type: 'category'}
             });
 
-            expect(gd._fullLayout.xaxis._categories).toEqual(['a', 'b', 1]);
+            expect(gd._fullLayout.xaxis._categories).toEqual(['a', 'b', '1']);
             expect(gd._fullLayout.xaxis._categoriesMap).toEqual({
                 '1': 2,
                 'a': 0,
diff --git a/test/jasmine/tests/contour_test.js b/test/jasmine/tests/contour_test.js
index b695fa3fa5d..7197f63c408 100644
--- a/test/jasmine/tests/contour_test.js
+++ b/test/jasmine/tests/contour_test.js
@@ -183,17 +183,28 @@ describe('contour makeColorMap', function() {
 describe('contour calc', function() {
     'use strict';
 
-    function _calc(opts) {
+    function _calc(opts, layout) {
         var base = { type: 'contour' };
         var trace = Lib.extendFlat({}, base, opts);
         var gd = { data: [trace] };
+        if(layout) gd.layout = layout;
 
         supplyAllDefaults(gd);
         var fullTrace = gd._fullData[0];
+        var fullLayout = gd._fullLayout;
         fullTrace._extremes = {};
 
+        // we used to call ax.setScale during supplyDefaults, and this had a
+        // fallback to provide _categories and _categoriesMap. Now neither of
+        // those is true... anyway the right way to do this though is
+        // ax.clearCalc.
+        fullLayout.xaxis.clearCalc();
+        fullLayout.yaxis.clearCalc();
+
         var out = Contour.calc(gd, fullTrace)[0];
         out.trace = fullTrace;
+        out._xcategories = fullLayout.xaxis._categories;
+        out._ycategories = fullLayout.yaxis._categories;
         return out;
     }
 
@@ -343,6 +354,57 @@ describe('contour calc', function() {
             });
         });
     });
+
+    ['contour'].forEach(function(traceType) {
+        it('should sort z data based on axis categoryorder for ' + traceType, function() {
+            var mock = require('@mocks/heatmap_categoryorder');
+            var mockCopy = Lib.extendDeep({}, mock);
+            var data = mockCopy.data[0];
+            data.type = traceType;
+            var layout = mockCopy.layout;
+
+            // sort x axis categories
+            var mockLayout = Lib.extendDeep({}, layout);
+            var out = _calc(data, mockLayout);
+            mockLayout.xaxis.categoryorder = 'category ascending';
+            var out1 = _calc(data, mockLayout);
+
+            expect(out._xcategories).toEqual(out1._xcategories.slice().reverse());
+            // Check z data is also sorted
+            for(var i = 0; i < out.z.length; i++) {
+                expect(out1.z[i]).toEqual(out.z[i].slice().reverse());
+            }
+
+            // sort y axis categories
+            mockLayout = Lib.extendDeep({}, layout);
+            out = _calc(data, mockLayout);
+            mockLayout.yaxis.categoryorder = 'category ascending';
+            out1 = _calc(data, mockLayout);
+
+            expect(out._ycategories).toEqual(out1._ycategories.slice().reverse());
+            // Check z data is also sorted
+            expect(out1.z).toEqual(out.z.slice().reverse());
+        });
+
+        it('should sort z data based on axis categoryarray ' + traceType, function() {
+            var mock = require('@mocks/heatmap_categoryorder');
+            var mockCopy = Lib.extendDeep({}, mock);
+            var data = mockCopy.data[0];
+            data.type = traceType;
+            var layout = mockCopy.layout;
+
+            layout.xaxis.categoryorder = 'array';
+            layout.xaxis.categoryarray = ['x', 'z', 'y', 'w'];
+            layout.yaxis.categoryorder = 'array';
+            layout.yaxis.categoryarray = ['a', 'd', 'b', 'c'];
+
+            var out = _calc(data, layout);
+
+            expect(out._xcategories).toEqual(layout.xaxis.categoryarray, 'xaxis should reorder');
+            expect(out._ycategories).toEqual(layout.yaxis.categoryarray, 'yaxis should reorder');
+            expect(out.z[0][0]).toEqual(0);
+        });
+    });
 });
 
 describe('contour plotting and editing', function() {
diff --git a/test/jasmine/tests/heatmap_test.js b/test/jasmine/tests/heatmap_test.js
index 961db01764e..aa85f05d7bd 100644
--- a/test/jasmine/tests/heatmap_test.js
+++ b/test/jasmine/tests/heatmap_test.js
@@ -491,6 +491,57 @@ describe('heatmap calc', function() {
         expect(out.y).toBeCloseToArray([0, 4, 8]);
         expect(out.z).toBeCloseTo2DArray([[1, 2, 3], [3, 1, 2]]);
     });
+
+    ['heatmap', 'heatmapgl'].forEach(function(traceType) {
+        it('should sort z data based on axis categoryorder for ' + traceType, function() {
+            var mock = require('@mocks/heatmap_categoryorder');
+            var mockCopy = Lib.extendDeep({}, mock);
+            var data = mockCopy.data[0];
+            data.type = traceType;
+            var layout = mockCopy.layout;
+
+            // sort x axis categories
+            var mockLayout = Lib.extendDeep({}, layout);
+            var out = _calc(data, mockLayout);
+            mockLayout.xaxis.categoryorder = 'category ascending';
+            var out1 = _calc(data, mockLayout);
+
+            expect(out._xcategories).toEqual(out1._xcategories.slice().reverse());
+            // Check z data is also sorted
+            for(var i = 0; i < out.z.length; i++) {
+                expect(out1.z[i]).toEqual(out.z[i].slice().reverse());
+            }
+
+            // sort y axis categories
+            mockLayout = Lib.extendDeep({}, layout);
+            out = _calc(data, mockLayout);
+            mockLayout.yaxis.categoryorder = 'category ascending';
+            out1 = _calc(data, mockLayout);
+
+            expect(out._ycategories).toEqual(out1._ycategories.slice().reverse());
+            // Check z data is also sorted
+            expect(out1.z).toEqual(out.z.slice().reverse());
+        });
+
+        it('should sort z data based on axis categoryarray ' + traceType, function() {
+            var mock = require('@mocks/heatmap_categoryorder');
+            var mockCopy = Lib.extendDeep({}, mock);
+            var data = mockCopy.data[0];
+            data.type = traceType;
+            var layout = mockCopy.layout;
+
+            layout.xaxis.categoryorder = 'array';
+            layout.xaxis.categoryarray = ['x', 'z', 'y', 'w'];
+            layout.yaxis.categoryorder = 'array';
+            layout.yaxis.categoryarray = ['a', 'd', 'b', 'c'];
+
+            var out = _calc(data, layout);
+
+            expect(out._xcategories).toEqual(layout.xaxis.categoryarray, 'xaxis should reorder');
+            expect(out._ycategories).toEqual(layout.yaxis.categoryarray, 'yaxis should reorder');
+            expect(out.z[0][0]).toEqual(0);
+        });
+    });
 });
 
 describe('heatmap plot', function() {
@@ -730,6 +781,24 @@ describe('heatmap hover', function() {
         });
     });
 
+    describe('with sorted categories', function() {
+        beforeAll(function(done) {
+            gd = createGraphDiv();
+
+            var mock = require('@mocks/heatmap_categoryorder.json');
+            var mockCopy = Lib.extendDeep({}, mock);
+
+            Plotly.plot(gd, mockCopy.data, mockCopy.layout).then(done);
+        });
+        afterAll(destroyGraphDiv);
+
+        it('should find closest point (case 1) and should', function() {
+            var pt = _hover(gd, 3, 1)[0];
+            expect(pt.index).toEqual([1, 3], 'have correct index');
+            assertLabels(pt, 2.5, 0.5, 0);
+        });
+    });
+
     describe('for xyz-column traces', function() {
         beforeAll(function(done) {
             gd = createGraphDiv();
diff --git a/test/jasmine/tests/histogram2d_test.js b/test/jasmine/tests/histogram2d_test.js
index 5a529fb548f..8188745b4ae 100644
--- a/test/jasmine/tests/histogram2d_test.js
+++ b/test/jasmine/tests/histogram2d_test.js
@@ -116,16 +116,29 @@ describe('Test histogram2d', function() {
 
 
     describe('calc', function() {
-        function _calc(opts) {
+        function _calc(opts, layout) {
             var base = { type: 'histogram2d' };
             var trace = Lib.extendFlat({}, base, opts);
             var gd = { data: [trace] };
+            if(layout) gd.layout = layout;
 
             supplyAllDefaults(gd);
             var fullTrace = gd._fullData[0];
+            var fullLayout = gd._fullLayout;
+
+            fullTrace._extremes = {};
+
+            // we used to call ax.setScale during supplyDefaults, and this had a
+            // fallback to provide _categories and _categoriesMap. Now neither of
+            // those is true... anyway the right way to do this though is
+            // ax.clearCalc.
+            fullLayout.xaxis.clearCalc();
+            fullLayout.yaxis.clearCalc();
 
             var out = calc(gd, fullTrace);
-            delete out.trace;
+            out._xcategories = fullLayout.xaxis._categories;
+            out._ycategories = fullLayout.yaxis._categories;
+
             return out;
         }
 
@@ -157,6 +170,60 @@ describe('Test histogram2d', function() {
                 [0, 0, 0, 1]
             ]);
         });
+
+        ['histogram2d', 'histogram2dcontour'].forEach(function(traceType) {
+            it('should sort z data based on axis categoryorder for ' + traceType, function() {
+                var mock = require('@mocks/heatmap_categoryorder');
+                var mockCopy = Lib.extendDeep({}, mock);
+                var data = mockCopy.data[0];
+                data.type = traceType;
+                var layout = mockCopy.layout;
+
+                // sort x axis categories
+                var mockLayout = Lib.extendDeep({}, layout);
+                var out = _calc(data, mockLayout);
+                mockLayout.xaxis.categoryorder = 'category ascending';
+                var out1 = _calc(data, mockLayout);
+
+                expect(out._xcategories).toEqual(out1._xcategories.slice().reverse());
+                // Check z data is also sorted
+                for(var i = 0; i < out.z.length; i++) {
+                    expect(out1.z[i]).toEqual(out.z[i].slice().reverse());
+                }
+
+                // sort y axis categories
+                mockLayout = Lib.extendDeep({}, layout);
+                out = _calc(data, mockLayout);
+                mockLayout.yaxis.categoryorder = 'category ascending';
+                out1 = _calc(data, mockLayout);
+
+                expect(out._ycategories).toEqual(out1._ycategories.slice().reverse());
+                // Check z data is also sorted
+                expect(out1.z).toEqual(out.z.slice().reverse());
+            });
+
+            it('should sort z data based on axis categoryarray ' + traceType, function() {
+                var mock = require('@mocks/heatmap_categoryorder');
+                var mockCopy = Lib.extendDeep({}, mock);
+                var data = mockCopy.data[0];
+                data.type = traceType;
+                var layout = mockCopy.layout;
+
+                layout.xaxis.categoryorder = 'array';
+                layout.xaxis.categoryarray = ['x', 'z', 'y', 'w'];
+                layout.yaxis.categoryorder = 'array';
+                layout.yaxis.categoryarray = ['a', 'd', 'b', 'c'];
+
+                var out = _calc(data, layout);
+
+                expect(out._xcategories).toEqual(layout.xaxis.categoryarray, 'xaxis should reorder');
+                expect(out._ycategories).toEqual(layout.yaxis.categoryarray, 'yaxis should reorder');
+                var offset = 0;
+                if(traceType === 'histogram2dcontour') offset = 1;
+                expect(out.z[0 + offset][0 + offset]).toEqual(0);
+                expect(out.z[0 + offset][3 + offset]).toEqual(1);
+            });
+        });
     });
 
     describe('restyle / relayout interaction', function() {