use language with variants with biggest count of simulations. other s…

…hould be removed.

lib/types.ts

@@ -16,6 +16,7 @@ export type Simulation = {
 
 export type LanguageDescriptor = {
   slug: string
+  langCode: string
   name: string
   localName: string
   url: string

steps/export.ts

@@ -9,7 +9,7 @@ import * as cheerio from 'cheerio'
 import { ZimArticle, ZimCreator } from '@openzim/libzim'
 
 import { log } from '../lib/logger.js'
-import { Target } from '../lib/types.js'
+import { Target, LanguageDescriptor, LanguageItemPair } from '../lib/types.js'
 import welcome from '../lib/welcome.js'
 import { Catalog } from '../lib/classes.js'
 import { getISO6393 } from '../lib/common.js'
@@ -30,7 +30,7 @@ const { argv } = yargs(hideBin(process.argv)).array('includeLanguages').array('e
 const langsFile = await fs.promises.readFile(path.join(__dirname, '../state/get/languages.json'))
 const langs = JSON.parse(langsFile.toString())
 
-const languages = Object.entries(langs).reduce((acc, [key, value]) => {
+const languages: LanguageItemPair<LanguageDescriptor> = Object.entries(langs).reduce((acc, [key, value]) => {
   if (argv.includeLanguages && !((argv.includeLanguages as string[]) || []).includes(key)) return acc
   if (argv.excludeLanguages && ((argv.excludeLanguages as string[]) || []).includes(key)) return acc // noinspection RedundantIfStatementJS
   return { ...acc, [key]: value }
@@ -257,11 +257,11 @@ const exportData = async () => {
     },
   ]
   if (!argv.mulOnly) {
-    for (const lang of Object.keys(languages)) {
+    for (const { langCode, slug } of Object.values(languages)) {
       targets.push({
-        output: `phet_${lang.toLowerCase().replace('_', '-')}_all_${datePostfix}`,
+        output: `phet_${langCode.toLowerCase().replace('_', '-')}_all_${datePostfix}`,
         date: now,
-        languages: [lang],
+        languages: [slug],
       })
     }
   }

steps/get.ts

@@ -81,10 +81,11 @@ const fetchLanguages = async (): Promise<void> => {
   rows.forEach((item) => {
     const url = $(item).find('td.list-highlight-background:first-child a').attr('href')
     const slug = /locale=(.*)$/.exec(url)?.pop()
-    if (argv.withoutLanguageVariants && slug.includes('_')) {
-      log.info(`Skipping ${slug} language`)
-      return
+
+    if (!getISO6393(slug)) {
+      throw new Error(`Failed to map language "${slug}" into ISO639-3.`)
     }
+
     const name = $(item).find('td.list-highlight-background:first-child a span').text()
 
     const nativeLangName = ISO6391.getNativeName(slug)
@@ -95,12 +96,27 @@ const fetchLanguages = async (): Promise<void> => {
     if (argv.includeLanguages && !((argv.includeLanguages as string[]) || []).includes(slug)) return
     if (argv.excludeLanguages && ((argv.excludeLanguages as string[]) || []).includes(slug)) return
 
-    if (!Object.keys(languages)?.includes(slug)) {
-      op.set(languages, slug, { slug, name, localName, url, count })
+    if (argv.withoutLanguageVariants && slug.includes('_')) {
+      const langCode = slug.split('_')[0]
+      if (slug === 'zh_CN') {
+        log.info(`Using ${slug} simulations for ${langCode} language`)
+        op.set(languages, slug, { slug, langCode, name, localName, url, count })
+        return
+      }
+
+      const existedLanguageKey = Object.keys(languages).find((language) => language.split('_')[0] === langCode)
+      if (existedLanguageKey && languages[existedLanguageKey].count < count) {
+        delete languages[existedLanguageKey]
+        log.info(`Using ${slug} simulations for ${langCode} language`)
+        op.set(languages, slug, { slug, langCode, name, localName, url, count })
+      } else {
+        log.info(`Skipping ${slug} language`)
+      }
+      return
     }
 
-    if (!getISO6393(slug)) {
-      throw new Error(`Failed to map language "${slug}" into ISO639-3.`)
+    if (!Object.keys(languages)?.includes(slug)) {
+      op.set(languages, slug, { slug, name, localName, url, count, langCode: slug })
     }
   })
   try {