feat(min_length): add minlength option for loading resources (#18)

pelias · May 14, 2019 · 1522b2a · 1522b2a
1 parent 2b6edfc
commit 1522b2a
Show file tree

Hide file tree

Showing 4 changed files with 25 additions and 13 deletions.
diff --git a/classifier/CompoundStreetClassifier.js b/classifier/CompoundStreetClassifier.js
@@ -9,15 +9,12 @@ class CompoundStreetClassifier extends WordClassifier {
   setup () {
     // load street suffixes
     this.suffixes = {}
-    libpostal.load(this.suffixes, ['de'], 'concatenated_suffixes_separable.txt')
 
-    // remove any suffixes which contain less than 3 characters (excluding a period)
-    // this removes suffixes such as 'r.' which can be ambiguous
-    for (let token in this.suffixes) {
-      if (token.length < 3) {
-        delete this.suffixes[token]
-      }
-    }
+    libpostal.load(this.suffixes, ['de'], 'concatenated_suffixes_separable.txt', {
+      // remove any suffixes which contain less than 3 characters (excluding a period)
+      // this removes suffixes such as 'r.' which can be ambiguous
+      minlength: 3
+    })
 
     // whitelist
     this.suffixes.park = true

diff --git a/classifier/PersonalTitleClassifier.js b/classifier/PersonalTitleClassifier.js
@@ -9,7 +9,8 @@ class PersonalTitleClassifier extends PhraseClassifier {
   setup () {
     this.index = {}
     libpostal.load(this.index, libpostal.languages, 'personal_titles.txt', {
-      replace: [/\.$/, '']
+      replace: [/\.$/, ''],
+      minlength: 2
     })
   }
 

diff --git a/resources/libpostal/libpostal.js b/resources/libpostal/libpostal.js
@@ -27,6 +27,9 @@ function _normalize (cell, options) {
   if (options && options.replace) {
     value = value.replace(options.replace[0], options.replace[1])
   }
+  if (options && options.minlength) {
+    if (value.length < options.minlength) { return '' }
+  }
   if (options && options.lowercase) {
     value = value.toLowerCase()
   }
@@ -36,14 +39,18 @@ function _normalize (cell, options) {
 function _add (index, options) {
   return cell => {
     const value = _normalize(cell, options)
-    index[value] = true
+    if (value && value.length) {
+      index[value] = true
+    }
   }
 }
 
 function _remove (index, options) {
   return cell => {
     const value = _normalize(cell, options)
-    delete index[value]
+    if (value && value.length) {
+      delete index[value]
+    }
   }
 }
 

diff --git a/resources/whosonfirst/whosonfirst.js b/resources/whosonfirst/whosonfirst.js
@@ -31,6 +31,9 @@ function _normalize (cell, options) {
   if (options && options.replace) {
     value = value.replace(options.replace[0], options.replace[1])
   }
+  if (options && options.minlength) {
+    if (value.length < options.minlength) { return '' }
+  }
   if (options && options.lowercase) {
     value = value.toLowerCase()
   }
@@ -40,14 +43,18 @@ function _normalize (cell, options) {
 function _add (set, options) {
   return cell => {
     const value = _normalize(cell, options)
-    set.add(value)
+    if (value && value.length) {
+      set.add(value)
+    }
   }
 }
 
 function _remove (set, options) {
   return cell => {
     const value = _normalize(cell, options)
-    set.delete(value)
+    if (value && value.length) {
+      set.delete(value)
+    }
   }
 }