Support x-ray-parse filters

index.js

@@ -26,8 +26,10 @@ var CONST = {
   }
 }
 
-function Xray () {
+function Xray (options) {
   var crawler = Crawler()
+  options = options || {}
+  var filters = options.filters || {}
 
   function xray (source, scope, selector) {
     var args = params(source, scope, selector)
@@ -40,7 +42,7 @@ function Xray () {
     var pages = []
     var stream
 
-    var walkHTML = WalkHTML(xray, selector, scope)
+    var walkHTML = WalkHTML(xray, selector, scope, filters)
     var request = Request(crawler)
 
     function node (source2, fn) {
@@ -65,7 +67,7 @@ function Xray () {
         })
       } else if (scope && ~scope.indexOf('@')) {
         debug('resolving to a url: %s', scope)
-        var url = resolve(source, false, scope)
+        var url = resolve(source, false, scope, filters)
 
         // ensure that a@href is a URL
         if (!isUrl(url)) {
@@ -111,7 +113,7 @@ function Xray () {
             return fn(null, pages)
           }
 
-          var url = resolve($, false, paginate)
+          var url = resolve($, false, paginate, filters)
           debug('paginate(%j) => %j', paginate, url)
 
           if (!isUrl(url)) {
@@ -198,11 +200,11 @@ function load (html, url) {
   return $
 }
 
-function WalkHTML (xray, selector, scope) {
+function WalkHTML (xray, selector, scope, filters) {
   return function walkHTML ($, fn) {
     walk(selector, function (v, k, next) {
       if (typeof v === 'string') {
-        var value = resolve($, root(scope), v)
+        var value = resolve($, root(scope), v, filters)
         return next(null, value)
       } else if (typeof v === 'function') {
         return v($, function (err, obj) {
@@ -211,7 +213,7 @@ function WalkHTML (xray, selector, scope) {
         })
       } else if (isArray(v)) {
         if (typeof v[0] === 'string') {
-          return next(null, resolve($, root(scope), v))
+          return next(null, resolve($, root(scope), v, filters))
         } else if (typeof v[0] === 'object') {
           var $scope = $.find ? $.find(scope) : $(scope)
           var pending = $scope.length

lib/resolve.js

@@ -24,6 +24,7 @@ module.exports = resolve
 
 function resolve ($, scope, selector, filters) {
   debug('resolve($j, %j)', scope, selector)
+  filters = filters || {}
   var array = isArray(selector)
   var obj = parse(array ? selector[0] : selector)
   obj.attribute = obj.attribute || 'text'
@@ -36,6 +37,14 @@ function resolve ($, scope, selector, filters) {
   var value = find($, scope, array ? [obj.selector] : obj.selector, obj.attribute)
   debug('resolved($j, %j) => %j', scope, selector, value)
 
+  if (array && typeof value.map === 'function') {
+    value = value.map(function (v) {
+      return filter(obj, $, scope, selector, v, filters)
+    })
+  } else {
+    value = filter(obj, $, scope, selector, value, filters)
+  }
+
   return value
 }
 
@@ -113,6 +122,32 @@ function attribute ($el, attr) {
   }
 }
 
+/**
+ * Filter the value(s)
+ *
+ * @param {Object} obj
+ * @param {Cheerio} $
+ * @param {String} scope
+ * @param {String|Array} selector
+ * @param {Object} filters
+ * @return {Array|String}
+ */
+
+function filter (obj, $, scope, selector, value, filters) {
+  var ctx = { $: $, selector: obj.selector, attribute: obj.attribute }
+  return (obj.filters || []).reduce(function (out, filter) {
+    var fn = filters[filter.name]
+    if (typeof fn === 'function') {
+      var args = [out].concat(filter.args || [])
+      var filtered = fn.apply(ctx, args)
+      debug('%s.apply(ctx, %j) => %j', filter.name, args, filtered)
+      return filtered
+    } else {
+      throw new Error('Invalid filter: ' + filter.name)
+    }
+  }, value)
+}
+
 // var cheerio = require('cheerio')
 // var $ = cheerio.load('<ul title="some title"><li>a</li><li>b</li></ul><ul><li>c</li><li>d</li></ul>')
 //

test/xray_spec.js

@@ -232,6 +232,36 @@ describe('Xray basics', function () {
     })
   })
 
+  it('should apply filters', function (done) {
+    var html = '<h3> All Tags </h3><ul class="tags"><li> a</li><li> b </li><li>c </li></ul><ul class="tags"><li>\nd</li><li>e</li></ul>'
+    var $ = cheerio.load(html)
+    var x = Xray({
+      filters: {
+        trim: function (value) {
+          return typeof value === 'string' ? value.trim() : value
+        },
+        slice: function (value, limit) {
+          return typeof value === 'string' ? value.slice(0, limit) : value
+        },
+        reverse: function (value) {
+          return typeof value === 'string' ? value.split('').reverse().join('') : value
+        }
+      }
+    })
+
+    x($, {
+      title: 'h3 | trim | reverse | slice:4',
+      tags: ['.tags > li | trim']
+    })(function (err, obj) {
+      if (err) return done(err)
+      assert.deepEqual(obj, {
+        title: 'sgaT',
+        tags: ['a', 'b', 'c', 'd', 'e']
+      })
+      done()
+    })
+  })
+
   // TODO: this could be tested better, need a static site
   // with pages
   it('should work with pagination & limits', function (done) {