geocoder.js 28 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790
  1. /**
  2. * @fileoverview Local reverse geocoder based on GeoNames data.
  3. * @author Thomas Steiner (tomac@google.com)
  4. * @license Apache 2.0
  5. *
  6. * @param {(object|object[])} points One single or an array of
  7. * latitude/longitude pairs
  8. * @param {integer} maxResults The maximum number of results to return
  9. * @callback callback The callback function with the results
  10. *
  11. * @returns {object[]} An array of GeoNames-based geocode results
  12. *
  13. * @example
  14. * // With just one point
  15. * var point = {latitude: 42.083333, longitude: 3.1};
  16. * geocoder.lookUp(point, 1, function(err, res) {
  17. * console.log(JSON.stringify(res, null, 2));
  18. * });
  19. *
  20. * // In batch mode with many points
  21. * var points = [
  22. * {latitude: 42.083333, longitude: 3.1},
  23. * {latitude: 48.466667, longitude: 9.133333}
  24. * ];
  25. * geocoder.lookUp(points, 1, function(err, res) {
  26. * console.log(JSON.stringify(res, null, 2));
  27. * });
  28. */
  29. 'use strict';
  30. var debug = require('debug')('local-reverse-geocoder');
  31. var fs = require('fs');
  32. var path = require('path');
  33. var parse = require('csv-parse');
  34. var kdTree = require('kdt');
  35. var request = require('request');
  36. var unzip = require('node-unzip-2');
  37. var async = require('async');
  38. var readline = require('readline');
  39. // All data from http://download.geonames.org/export/dump/
  40. var GEONAMES_URL = 'http://download.geonames.org/export/dump/';
  41. var CITIES_FILE = 'cities1000';
  42. var ADMIN_1_CODES_FILE = 'admin1CodesASCII';
  43. var ADMIN_2_CODES_FILE = 'admin2Codes';
  44. var ALL_COUNTRIES_FILE = 'allCountries';
  45. var ALTERNATE_NAMES_FILE = 'alternateNames';
  46. /* jshint maxlen: false */
  47. var GEONAMES_COLUMNS = [
  48. 'geoNameId', // integer id of record in geonames database
  49. 'name', // name of geographical point (utf8) varchar(200)
  50. 'asciiName', // name of geographical point in plain ascii characters, varchar(200)
  51. 'alternateNames', // alternatenames, comma separated, ascii names automatically transliterated, convenience attribute from alternatename table, varchar(10000)
  52. 'latitude', // latitude in decimal degrees (wgs84)
  53. 'longitude', // longitude in decimal degrees (wgs84)
  54. 'featureClass', // see http://www.geonames.org/export/codes.html, char(1)
  55. 'featureCode', // see http://www.geonames.org/export/codes.html, varchar(10)
  56. 'countryCode', // ISO-3166 2-letter country code, 2 characters
  57. 'cc2', // alternate country codes, comma separated, ISO-3166 2-letter country code, 60 characters
  58. 'admin1Code', // fipscode (subject to change to iso code), see exceptions below, see file admin1Codes.txt for display names of this code; varchar(20)
  59. 'admin2Code', // code for the second administrative division, a county in the US, see file admin2Codes.txt; varchar(80)
  60. 'admin3Code', // code for third level administrative division, varchar(20)
  61. 'admin4Code', // code for fourth level administrative division, varchar(20)
  62. 'population', // bigint (8 byte int)
  63. 'elevation', // in meters, integer
  64. 'dem', // digital elevation model, srtm3 or gtopo30, average elevation 3''x3'' (ca 90mx90m) or 30''x30'' (ca 900mx900m) area in meters, integer. srtm processed by cgiar/ciat.
  65. 'timezone', // the timezone id (see file timeZone.txt) varchar(40)
  66. 'modificationDate', // date of last modification in yyyy-MM-dd format
  67. ];
  68. /* jshint maxlen: 80 */
  69. var GEONAMES_ADMIN_CODES_COLUMNS = [
  70. 'concatenatedCodes',
  71. 'name',
  72. 'asciiName',
  73. 'geoNameId'
  74. ];
  75. /* jshint maxlen: false */
  76. var GEONAMES_ALTERNATE_NAMES_COLUMNS = [
  77. 'alternateNameId', // the id of this alternate name, int
  78. 'geoNameId', // geonameId referring to id in table 'geoname', int
  79. 'isoLanguage', // iso 639 language code 2- or 3-characters; 4-characters 'post' for postal codes and 'iata','icao' and faac for airport codes, fr_1793 for French Revolution name
  80. 'alternateNames', // alternate name or name variant, varchar(200)
  81. 'isPreferrredName', // '1', if this alternate name is an official/preferred name
  82. 'isShortName', // '1', if this is a short name like 'California' for 'State of California'
  83. 'isColloquial', // '1', if this alternate name is a colloquial or slang term
  84. 'isHistoric' // '1', if this alternate name is historic and was used in the past
  85. ];
  86. /* jshint maxlen: 80 */
  87. var GEONAMES_DUMP = __dirname + '/geonames_dump';
  88. var geocoder = {
  89. _kdTree: null,
  90. _admin1Codes: null,
  91. _admin2Codes: null,
  92. _admin3Codes: null,
  93. _admin4Codes: null,
  94. _alternateNames: null,
  95. // Distance function taken from
  96. // http://www.movable-type.co.uk/scripts/latlong.html
  97. _distanceFunc: function distance(x, y) {
  98. var toRadians = function(num) {
  99. return num * Math.PI / 180;
  100. };
  101. var lat1 = x.latitude;
  102. var lon1 = x.longitude;
  103. var lat2 = y.latitude;
  104. var lon2 = y.longitude;
  105. var R = 6371; // km
  106. var φ1 = toRadians(lat1);
  107. var φ2 = toRadians(lat2);
  108. var Δφ = toRadians(lat2 - lat1);
  109. var Δλ = toRadians(lon2 - lon1);
  110. var a = Math.sin(Δφ / 2) * Math.sin(Δφ / 2) +
  111. Math.cos(φ1) * Math.cos(φ2) *
  112. Math.sin(Δλ / 2) * Math.sin(Δλ / 2);
  113. var c = 2 * Math.atan2(Math.sqrt(a), Math.sqrt(1 - a));
  114. return R * c;
  115. },
  116. _getGeoNamesAlternateNamesData: function(callback) {
  117. var now = (new Date()).toISOString().substr(0, 10);
  118. // Use timestamped alternate names file OR bare alternate names file
  119. var timestampedFilename = GEONAMES_DUMP + '/alternate_names/' +
  120. ALTERNATE_NAMES_FILE + '_' + now + '.txt';
  121. if (fs.existsSync(timestampedFilename)) {
  122. debug('Using cached GeoNames alternate names data from ' +
  123. timestampedFilename);
  124. return callback(null, timestampedFilename);
  125. }
  126. var filename = GEONAMES_DUMP + '/alternate_names/' + ALTERNATE_NAMES_FILE +
  127. '.txt';
  128. if (fs.existsSync(filename)) {
  129. debug('Using cached GeoNames alternate names data from ' +
  130. filename);
  131. return callback(null, filename);
  132. }
  133. debug('Getting GeoNames alternate names data from ' +
  134. GEONAMES_URL + ALTERNATE_NAMES_FILE + '.zip (this may take a while)');
  135. var options = {
  136. url: GEONAMES_URL + ALTERNATE_NAMES_FILE + '.zip',
  137. encoding: null
  138. };
  139. request.get(options, function(err, response, body) {
  140. if (err || response.statusCode !== 200) {
  141. return callback('Error downloading GeoNames alternate names data' +
  142. (err ? ': ' + err : ''));
  143. }
  144. debug('Received zipped GeoNames alternate names data');
  145. // Store a dump locally
  146. if (!fs.existsSync(GEONAMES_DUMP + '/alternate_names')) {
  147. fs.mkdirSync(GEONAMES_DUMP + '/alternate_names');
  148. }
  149. var zipFilename = GEONAMES_DUMP + '/alternate_names/' +
  150. ALTERNATE_NAMES_FILE + '_' + now + '.zip';
  151. try {
  152. fs.writeFileSync(zipFilename, body);
  153. fs.createReadStream(zipFilename)
  154. .pipe(unzip.Extract({path: GEONAMES_DUMP + '/alternate_names'}))
  155. .on('error', function(e) {
  156. console.error(e);
  157. })
  158. .on('close', function() {
  159. fs.renameSync(filename, timestampedFilename);
  160. fs.unlinkSync(GEONAMES_DUMP + '/alternate_names/' +
  161. ALTERNATE_NAMES_FILE + '_' + now + '.zip');
  162. debug('Unzipped GeoNames alternate names data');
  163. // Housekeeping, remove old files
  164. var currentFileName = path.basename(timestampedFilename);
  165. fs.readdirSync(GEONAMES_DUMP + '/alternate_names').forEach(
  166. function(file) {
  167. if (file !== currentFileName) {
  168. fs.unlinkSync(GEONAMES_DUMP + '/alternate_names/' + file);
  169. }
  170. });
  171. return callback(null, timestampedFilename);
  172. });
  173. } catch (e) {
  174. debug('Warning: ' + e);
  175. return callback(null, timestampedFilename);
  176. }
  177. });
  178. },
  179. _parseGeoNamesAlternateNamesCsv: function(pathToCsv, callback) {
  180. var that = this;
  181. that._alternateNames = {};
  182. var lineReader = readline.createInterface({
  183. input: fs.createReadStream(pathToCsv)
  184. });
  185. lineReader.on('line', function(line) {
  186. line = line.split('\t');
  187. const [
  188. _,
  189. geoNameId,
  190. isoLanguage,
  191. altName,
  192. isPreferredName,
  193. isShortName,
  194. isColloquial,
  195. isHistoric
  196. ] = line;
  197. if (isoLanguage === '') {
  198. // consider data without country code as invalid
  199. return;
  200. }
  201. if (!that._alternateNames[geoNameId]) {
  202. that._alternateNames[geoNameId] = {};
  203. }
  204. that._alternateNames[geoNameId][isoLanguage] = {
  205. altName,
  206. isPreferredName: Boolean(isPreferredName),
  207. isShortName: Boolean(isShortName),
  208. isColloquial: Boolean(isColloquial),
  209. isHistoric: Boolean(isHistoric)
  210. };
  211. });
  212. lineReader.on('close', function() {
  213. return callback();
  214. });
  215. },
  216. _getGeoNamesAdmin1CodesData: function(callback) {
  217. var now = (new Date()).toISOString().substr(0, 10);
  218. var timestampedFilename = GEONAMES_DUMP + '/admin1_codes/' +
  219. ADMIN_1_CODES_FILE + '_' + now + '.txt';
  220. if (fs.existsSync(timestampedFilename)) {
  221. debug('Using cached GeoNames admin 1 codes data from ' +
  222. timestampedFilename);
  223. return callback(null, timestampedFilename);
  224. }
  225. var filename = GEONAMES_DUMP + '/admin1_codes/' + ADMIN_1_CODES_FILE +
  226. '.txt';
  227. if (fs.existsSync(filename)) {
  228. debug('Using cached GeoNames admin 1 codes data from ' +
  229. filename);
  230. return callback(null, filename);
  231. }
  232. debug('Getting GeoNames admin 1 codes data from ' +
  233. GEONAMES_URL + ADMIN_1_CODES_FILE + '.txt (this may take a while)');
  234. var url = GEONAMES_URL + ADMIN_1_CODES_FILE + '.txt';
  235. request.get(url, function(err, response, body) {
  236. if (err || response.statusCode !== 200) {
  237. return callback('Error downloading GeoNames admin 1 codes data' +
  238. (err ? ': ' + err : ''));
  239. }
  240. // Store a dump locally
  241. if (!fs.existsSync(GEONAMES_DUMP + '/admin1_codes')) {
  242. fs.mkdirSync(GEONAMES_DUMP + '/admin1_codes');
  243. }
  244. try {
  245. fs.writeFileSync(timestampedFilename, body);
  246. // Housekeeping, remove old files
  247. var currentFileName = path.basename(timestampedFilename);
  248. fs.readdirSync(GEONAMES_DUMP + '/admin1_codes').forEach(function(file) {
  249. if (file !== currentFileName) {
  250. fs.unlinkSync(GEONAMES_DUMP + '/admin1_codes/' + file);
  251. }
  252. });
  253. } catch (e) {
  254. throw(e);
  255. }
  256. return callback(null, timestampedFilename);
  257. });
  258. },
  259. _parseGeoNamesAdmin1CodesCsv: function(pathToCsv, callback) {
  260. var that = this;
  261. var lenI = GEONAMES_ADMIN_CODES_COLUMNS.length;
  262. that._admin1Codes = {};
  263. var lineReader = readline.createInterface({
  264. input: fs.createReadStream(pathToCsv)
  265. });
  266. lineReader.on('line', function(line) {
  267. line = line.split('\t');
  268. for (var i = 0; i < lenI; i++) {
  269. var value = line[i] || null;
  270. if (i === 0) {
  271. that._admin1Codes[value] = {};
  272. } else {
  273. that._admin1Codes[line[0]][GEONAMES_ADMIN_CODES_COLUMNS[i]] = value;
  274. }
  275. }
  276. });
  277. lineReader.on('close', function() {
  278. return callback();
  279. });
  280. },
  281. _getGeoNamesAdmin2CodesData: function(callback) {
  282. var now = (new Date()).toISOString().substr(0, 10);
  283. var timestampedFilename = GEONAMES_DUMP + '/admin2_codes/' +
  284. ADMIN_2_CODES_FILE + '_' + now + '.txt';
  285. if (fs.existsSync(timestampedFilename)) {
  286. debug('Using cached GeoNames admin 2 codes data from ' +
  287. timestampedFilename);
  288. return callback(null, timestampedFilename);
  289. }
  290. var filename = GEONAMES_DUMP + '/admin2_codes/' + ADMIN_2_CODES_FILE +
  291. '.txt';
  292. if (fs.existsSync(filename)) {
  293. debug('Using cached GeoNames admin 2 codes data from ' +
  294. filename);
  295. return callback(null, filename);
  296. }
  297. debug('Getting GeoNames admin 2 codes data from ' +
  298. GEONAMES_URL + ADMIN_2_CODES_FILE + '.txt (this may take a while)');
  299. var url = GEONAMES_URL + ADMIN_2_CODES_FILE + '.txt';
  300. request.get(url, function(err, response, body) {
  301. if (err || response.statusCode !== 200) {
  302. return callback('Error downloading GeoNames admin 2 codes data' +
  303. (err ? ': ' + err : ''));
  304. }
  305. // Store a dump locally
  306. if (!fs.existsSync(GEONAMES_DUMP + '/admin2_codes')) {
  307. fs.mkdirSync(GEONAMES_DUMP + '/admin2_codes');
  308. }
  309. try {
  310. fs.writeFileSync(timestampedFilename, body);
  311. // Housekeeping, remove old files
  312. var currentFileName = path.basename(timestampedFilename);
  313. fs.readdirSync(GEONAMES_DUMP + '/admin2_codes').forEach(function(file) {
  314. if (file !== currentFileName) {
  315. fs.unlinkSync(GEONAMES_DUMP + '/admin2_codes/' + file);
  316. }
  317. });
  318. } catch (e) {
  319. throw(e);
  320. }
  321. return callback(null, timestampedFilename);
  322. });
  323. },
  324. _parseGeoNamesAdmin2CodesCsv: function(pathToCsv, callback) {
  325. var that = this;
  326. var lenI = GEONAMES_ADMIN_CODES_COLUMNS.length;
  327. that._admin2Codes = {};
  328. var lineReader = readline.createInterface({
  329. input: fs.createReadStream(pathToCsv)
  330. });
  331. lineReader.on('line', function(line) {
  332. line = line.split('\t');
  333. for (var i = 0; i < lenI; i++) {
  334. var value = line[i] || null;
  335. if (i === 0) {
  336. that._admin2Codes[value] = {};
  337. } else {
  338. that._admin2Codes[line[0]][GEONAMES_ADMIN_CODES_COLUMNS[i]] = value;
  339. }
  340. }
  341. });
  342. lineReader.on('close', function() {
  343. return callback();
  344. });
  345. },
  346. _getGeoNamesCitiesData: function(callback) {
  347. var now = (new Date()).toISOString().substr(0, 10);
  348. // Use timestamped cities file OR bare cities file
  349. var timestampedFilename = GEONAMES_DUMP + '/cities/' + CITIES_FILE + '_' +
  350. now + '.txt';
  351. if (fs.existsSync(timestampedFilename)) {
  352. debug('Using cached GeoNames cities data from ' +
  353. timestampedFilename);
  354. return callback(null, timestampedFilename);
  355. }
  356. var filename = GEONAMES_DUMP + '/cities/' + CITIES_FILE + '.txt';
  357. if (fs.existsSync(filename)) {
  358. debug('Using cached GeoNames cities data from ' +
  359. filename);
  360. return callback(null, filename);
  361. }
  362. debug('Getting GeoNames cities data from ' + GEONAMES_URL +
  363. CITIES_FILE + '.zip (this may take a while)');
  364. var options = {
  365. url: GEONAMES_URL + CITIES_FILE + '.zip',
  366. encoding: null
  367. };
  368. request.get(options, function(err, response, body) {
  369. if (err || response.statusCode !== 200) {
  370. return callback('Error downloading GeoNames cities data' +
  371. (err ? ': ' + err : ''));
  372. }
  373. debug('Received zipped GeoNames cities data');
  374. // Store a dump locally
  375. if (!fs.existsSync(GEONAMES_DUMP + '/cities')) {
  376. fs.mkdirSync(GEONAMES_DUMP + '/cities');
  377. }
  378. var zipFilename = GEONAMES_DUMP + '/cities/' + CITIES_FILE + '_' + now +
  379. '.zip';
  380. try {
  381. fs.writeFileSync(zipFilename, body);
  382. fs.createReadStream(zipFilename)
  383. .pipe(unzip.Extract({path: GEONAMES_DUMP + '/cities'}))
  384. .on('close', function() {
  385. fs.renameSync(filename, timestampedFilename);
  386. fs.unlinkSync(GEONAMES_DUMP + '/cities/' + CITIES_FILE + '_' + now +
  387. '.zip');
  388. debug('Unzipped GeoNames cities data');
  389. // Housekeeping, remove old files
  390. var currentFileName = path.basename(timestampedFilename);
  391. fs.readdirSync(GEONAMES_DUMP + '/cities').forEach(function(file) {
  392. if (file !== currentFileName) {
  393. fs.unlinkSync(GEONAMES_DUMP + '/cities/' + file);
  394. }
  395. });
  396. return callback(null, timestampedFilename);
  397. });
  398. } catch (e) {
  399. debug('Warning: ' + e);
  400. return callback(null, timestampedFilename);
  401. }
  402. });
  403. },
  404. _parseGeoNamesCitiesCsv: function(pathToCsv, callback) {
  405. debug('Started parsing cities.txt (this may take a ' +
  406. 'while)');
  407. var data = [];
  408. var lenI = GEONAMES_COLUMNS.length;
  409. var that = this;
  410. var content = fs.readFileSync(pathToCsv);
  411. parse(content, {delimiter: '\t', quote: ''}, function(err, lines) {
  412. if (err) {
  413. return callback(err);
  414. }
  415. lines.forEach(function(line) {
  416. var lineObj = {};
  417. for (var i = 0; i < lenI; i++) {
  418. var column = line[i] || null;
  419. lineObj[GEONAMES_COLUMNS[i]] = column;
  420. }
  421. data.push(lineObj);
  422. });
  423. debug('Finished parsing cities.txt');
  424. debug('Started building cities k-d tree (this may take ' +
  425. 'a while)');
  426. var dimensions = [
  427. 'latitude',
  428. 'longitude'
  429. ];
  430. that._kdTree = kdTree.createKdTree(data, that._distanceFunc, dimensions);
  431. debug('Finished building cities k-d tree');
  432. return callback();
  433. });
  434. },
  435. _getGeoNamesAllCountriesData: function(callback) {
  436. var now = (new Date()).toISOString().substr(0, 10);
  437. var timestampedFilename = GEONAMES_DUMP + '/all_countries/' +
  438. ALL_COUNTRIES_FILE + '_' + now + '.txt';
  439. if (fs.existsSync(timestampedFilename)) {
  440. debug('Using cached GeoNames all countries data from ' +
  441. timestampedFilename);
  442. return callback(null, timestampedFilename);
  443. }
  444. var filename = GEONAMES_DUMP + '/all_countries/' + ALL_COUNTRIES_FILE +
  445. '.txt';
  446. if (fs.existsSync(filename)) {
  447. debug('Using cached GeoNames all countries data from ' +
  448. filename);
  449. return callback(null, filename);
  450. }
  451. debug('Getting GeoNames all countries data from ' +
  452. GEONAMES_URL + ALL_COUNTRIES_FILE + '.zip (this may take a while)');
  453. var options = {
  454. url: GEONAMES_URL + ALL_COUNTRIES_FILE + '.zip',
  455. encoding: null
  456. };
  457. request.get(options, function(err, response, body) {
  458. if (err || response.statusCode !== 200) {
  459. return callback('Error downloading GeoNames all countries data' +
  460. (err ? ': ' + err : ''));
  461. }
  462. debug('Received zipped GeoNames all countries data');
  463. // Store a dump locally
  464. if (!fs.existsSync(GEONAMES_DUMP + '/all_countries')) {
  465. fs.mkdirSync(GEONAMES_DUMP + '/all_countries');
  466. }
  467. var zipFilename = GEONAMES_DUMP + '/all_countries/' + ALL_COUNTRIES_FILE +
  468. '_' + now + '.zip';
  469. try {
  470. fs.writeFileSync(zipFilename, body);
  471. fs.createReadStream(zipFilename)
  472. .pipe(unzip.Extract({path: GEONAMES_DUMP + '/all_countries'}))
  473. .on('close', function() {
  474. fs.renameSync(filename, timestampedFilename);
  475. fs.unlinkSync(GEONAMES_DUMP + '/all_countries/' +
  476. ALL_COUNTRIES_FILE + '_' + now + '.zip');
  477. debug('Unzipped GeoNames all countries data');
  478. // Housekeeping, remove old files
  479. var currentFileName = path.basename(timestampedFilename);
  480. var directory = GEONAMES_DUMP + '/all_countries';
  481. fs.readdirSync(directory).forEach(function(file) {
  482. if (file !== currentFileName) {
  483. fs.unlinkSync(GEONAMES_DUMP + '/all_countries/' + file);
  484. }
  485. });
  486. return callback(null, timestampedFilename);
  487. });
  488. } catch (e) {
  489. debug('Warning: ' + e);
  490. return callback(null, timestampedFilename);
  491. }
  492. });
  493. },
  494. _parseGeoNamesAllCountriesCsv: function(pathToCsv, callback) {
  495. debug('Started parsing all countries.txt (this may take ' +
  496. 'a while)');
  497. var lenI = GEONAMES_COLUMNS.length;
  498. var that = this;
  499. // Indexes
  500. var featureCodeIndex = GEONAMES_COLUMNS.indexOf('featureCode');
  501. var countryCodeIndex = GEONAMES_COLUMNS.indexOf('countryCode');
  502. var admin1CodeIndex = GEONAMES_COLUMNS.indexOf('admin1Code');
  503. var admin2CodeIndex = GEONAMES_COLUMNS.indexOf('admin2Code');
  504. var admin3CodeIndex = GEONAMES_COLUMNS.indexOf('admin3Code');
  505. var admin4CodeIndex = GEONAMES_COLUMNS.indexOf('admin4Code');
  506. var nameIndex = GEONAMES_COLUMNS.indexOf('name');
  507. var asciiNameIndex = GEONAMES_COLUMNS.indexOf('asciiName');
  508. var geoNameIdIndex = GEONAMES_COLUMNS.indexOf('geoNameId');
  509. var counter = 0;
  510. that._admin3Codes = {};
  511. that._admin4Codes = {};
  512. var lineReader = readline.createInterface({
  513. input: fs.createReadStream(pathToCsv)
  514. });
  515. lineReader.on('line', function(line) {
  516. line = line.split('\t');
  517. var featureCode = line[featureCodeIndex];
  518. if ((featureCode === 'ADM3') || (featureCode === 'ADM4')) {
  519. var lineObj = {
  520. name: line[nameIndex],
  521. asciiName: line[asciiNameIndex],
  522. geoNameId: line[geoNameIdIndex]
  523. };
  524. var key = line[countryCodeIndex] + '.' + line[admin1CodeIndex] + '.' +
  525. line[admin2CodeIndex] + '.' + line[admin3CodeIndex];
  526. if (featureCode === 'ADM3') {
  527. that._admin3Codes[key] = lineObj;
  528. } else if (featureCode === 'ADM4') {
  529. that._admin4Codes[key + '.' + line[admin4CodeIndex]] = lineObj;
  530. }
  531. }
  532. if (counter % 100000 === 0) {
  533. debug('Parsing progress all countries ' + counter);
  534. }
  535. counter++;
  536. });
  537. lineReader.on('close', function() {
  538. debug('Finished parsing all countries.txt');
  539. return callback();
  540. });
  541. },
  542. init: function(options, callback) {
  543. options = options || {};
  544. if (options.dumpDirectory) {
  545. GEONAMES_DUMP = options.dumpDirectory;
  546. }
  547. options.load = options.load || {};
  548. if (options.load.admin1 === undefined) {
  549. options.load.admin1 = true;
  550. }
  551. if (options.load.admin2 === undefined) {
  552. options.load.admin2 = true;
  553. }
  554. if (options.load.admin3And4 === undefined) {
  555. options.load.admin3And4 = true;
  556. }
  557. if (options.load.alternateNames === undefined) {
  558. options.load.alternateNames = true;
  559. }
  560. debug('Initializing local reverse geocoder using dump ' +
  561. 'directory: ' + GEONAMES_DUMP);
  562. // Create local cache folder
  563. if (!fs.existsSync(GEONAMES_DUMP)) {
  564. fs.mkdirSync(GEONAMES_DUMP);
  565. }
  566. var that = this;
  567. async.parallel([
  568. // Get GeoNames cities
  569. function(waterfallCallback) {
  570. async.waterfall([
  571. that._getGeoNamesCitiesData.bind(that),
  572. that._parseGeoNamesCitiesCsv.bind(that)
  573. ], function() {
  574. return waterfallCallback();
  575. });
  576. },
  577. // Get GeoNames admin 1 codes
  578. function(waterfallCallback) {
  579. if (options.load.admin1) {
  580. async.waterfall([
  581. that._getGeoNamesAdmin1CodesData.bind(that),
  582. that._parseGeoNamesAdmin1CodesCsv.bind(that)
  583. ], function() {
  584. return waterfallCallback();
  585. });
  586. } else {
  587. return setImmediate(waterfallCallback);
  588. }
  589. },
  590. // Get GeoNames admin 2 codes
  591. function(waterfallCallback) {
  592. if (options.load.admin2) {
  593. async.waterfall([
  594. that._getGeoNamesAdmin2CodesData.bind(that),
  595. that._parseGeoNamesAdmin2CodesCsv.bind(that)
  596. ], function() {
  597. return waterfallCallback();
  598. });
  599. } else {
  600. return setImmediate(waterfallCallback);
  601. }
  602. },
  603. // Get GeoNames all countries
  604. function(waterfallCallback) {
  605. if (options.load.admin3And4) {
  606. async.waterfall([
  607. that._getGeoNamesAllCountriesData.bind(that),
  608. that._parseGeoNamesAllCountriesCsv.bind(that)
  609. ], function() {
  610. return waterfallCallback();
  611. });
  612. } else {
  613. return setImmediate(waterfallCallback);
  614. }
  615. },
  616. // Get GeoNames alternate names
  617. function(waterfallCallback) {
  618. if (options.load.alternateNames) {
  619. async.waterfall([
  620. that._getGeoNamesAlternateNamesData.bind(that),
  621. that._parseGeoNamesAlternateNamesCsv.bind(that)
  622. ], function() {
  623. return waterfallCallback();
  624. });
  625. } else {
  626. return setImmediate(waterfallCallback);
  627. }
  628. }
  629. ],
  630. // Main callback
  631. function(err) {
  632. if (err) {
  633. throw(err);
  634. }
  635. return callback();
  636. });
  637. },
  638. lookUp: function(points, arg2, arg3) {
  639. var callback;
  640. var maxResults;
  641. if (arguments.length === 2) {
  642. maxResults = 1;
  643. callback = arg2;
  644. } else {
  645. maxResults = arg2;
  646. callback = arg3;
  647. }
  648. this._lookUp(points, maxResults, function(err, results) {
  649. return callback(null, results);
  650. });
  651. },
  652. _lookUp: function(points, maxResults, callback) {
  653. var that = this;
  654. // If not yet initialied, then initialize
  655. if (!this._kdTree) {
  656. return this.init({}, function() {
  657. return that.lookUp(points, maxResults, callback);
  658. });
  659. }
  660. // Make sure we have an array of points
  661. if (!Array.isArray(points)) {
  662. points = [points];
  663. }
  664. var functions = [];
  665. points.forEach(function(point, i) {
  666. point = {
  667. latitude: parseFloat(point.latitude),
  668. longitude: parseFloat(point.longitude)
  669. };
  670. debug('Look-up request for point ' +
  671. JSON.stringify(point));
  672. functions[i] = function(innerCallback) {
  673. var result = that._kdTree.nearest(point, maxResults);
  674. result.reverse();
  675. for (var j = 0, lenJ = result.length; j < lenJ; j++) {
  676. if (result && result[j] && result[j][0]) {
  677. var countryCode = result[j][0].countryCode || '';
  678. var geoNameId = result[j][0].geoNameId || '';
  679. var admin1Code;
  680. var admin2Code;
  681. var admin3Code;
  682. var admin4Code;
  683. // Look-up of admin 1 code
  684. if (that._admin1Codes) {
  685. admin1Code = result[j][0].admin1Code || '';
  686. var admin1CodeKey = countryCode + '.' + admin1Code;
  687. result[j][0].admin1Code = that._admin1Codes[admin1CodeKey] ||
  688. result[j][0].admin1Code;
  689. }
  690. // Look-up of admin 2 code
  691. if (that._admin2Codes) {
  692. admin2Code = result[j][0].admin2Code || '';
  693. var admin2CodeKey = countryCode + '.' + admin1Code + '.' +
  694. admin2Code;
  695. result[j][0].admin2Code = that._admin2Codes[admin2CodeKey] ||
  696. result[j][0].admin2Code;
  697. }
  698. // Look-up of admin 3 code
  699. if (that._admin3Codes) {
  700. admin3Code = result[j][0].admin3Code || '';
  701. var admin3CodeKey = countryCode + '.' + admin1Code + '.' +
  702. admin2Code + '.' + admin3Code;
  703. result[j][0].admin3Code = that._admin3Codes[admin3CodeKey] ||
  704. result[j][0].admin3Code;
  705. }
  706. // Look-up of admin 4 code
  707. if (that._admin4Codes) {
  708. admin4Code = result[j][0].admin4Code || '';
  709. var admin4CodeKey = countryCode + '.' + admin1Code + '.' +
  710. admin2Code + '.' + admin3Code + '.' + admin4Code;
  711. result[j][0].admin4Code = that._admin4Codes[admin4CodeKey] ||
  712. result[j][0].admin4Code;
  713. }
  714. // Look-up of alternate name
  715. if (that._alternateNames) {
  716. result[j][0].alternateName = that._alternateNames[geoNameId] ||
  717. result[j][0].alternateName;
  718. }
  719. // Pull in the k-d tree distance in the main object
  720. result[j][0].distance = result[j][1];
  721. // Simplify the output by not returning an array
  722. result[j] = result[j][0];
  723. }
  724. }
  725. debug('Found result(s) for point ' +
  726. JSON.stringify(point) + result.map(function(subResult, i) {
  727. return '\n (' + (++i) + ') {"geoNameId":"' +
  728. subResult.geoNameId + '",' + '"name":"' + subResult.name +
  729. '"}';
  730. }));
  731. return innerCallback(null, result);
  732. };
  733. });
  734. async.series(
  735. functions,
  736. function(err, results) {
  737. debug('Delivering joint results');
  738. return callback(null, results);
  739. });
  740. }
  741. };
  742. module.exports = geocoder;