Merge pull request #2752 from iptv-org/patch-2025.04.3

Patch 2025.04.3

Author: freearhey

CONTRIBUTING.md

@@ -391,9 +391,9 @@ npm run channels:edit path/to/channels.xml
 This way, you can map channels by simply selecting the proper ID from the list:
 
 ```sh
-? Select xmltv_id for "BBC One" (bbc1): (Use arrow keys)
-❯ BBC One (BBC1, BBC Television, BBC Television Service) | BBCOne.uk
-  BBC One HD | BBCOneHD.uk
+? Select channel ID for "BBC One" (bbc1): (Use arrow keys)
+❯ BBCOne.uk (BBC One, BBC1, BBC Television, BBC Television Service)
+  BBCOneHD.uk (BBC One HD)
   Type...
   Skip
 ```

README.md

@@ -57,7 +57,7 @@ Options:
   -c, --channels <path>         Path to *.channels.xml file (required if the "--site" attribute is
                                 not specified)
   -o, --output <path>           Path to output file (default: "guide.xml")
-  -l, --lang <code>             Allows to limit the download to channels in the specified language only (ISO 639-1 code)
+  -l, --lang <codes>            Allows you to restrict downloading to channels in specified languages only (example: "en,id")
   -t, --timeout <milliseconds>  Timeout for each request in milliseconds (default: 0)
   -d, --delay <milliseconds>    Delay between request in milliseconds (default: 0)
   -x, --proxy <url>             Use the specified proxy (example: "socks5://username:[email protected]:1234")

package-lock.json

@@ -43,6 +43,7 @@
     "@eslint/eslintrc": "^3.2.0",
     "@eslint/js": "^9.17.0",
     "@freearhey/core": "^0.7.0",
+    "@freearhey/search-js": "^0.1.2",
     "@ntlab/sfetch": "^1.2.0",
     "@octokit/core": "^6.1.3",
     "@octokit/plugin-paginate-rest": "^11.3.6",
@@ -74,7 +75,6 @@
     "eslint-config-prettier": "^9.0.0",
     "form-data": "^4.0.0",
     "fs-extra": "^10.0.1",
-    "fuse.js": "^7.0.0",
     "glob": "^7.2.0",
     "globals": "^15.14.0",
     "husky": "^9.1.7",

package.json

@@ -1,19 +1,24 @@
-import { Logger } from '@freearhey/core'
-import { ApiClient } from '../../core'
+import { DATA_DIR } from '../../constants'
+import { Storage } from '@freearhey/core'
+import { DataLoader } from '../../core'
 
 async function main() {
-  const logger = new Logger()
-  const client = new ApiClient({ logger })
+  const storage = new Storage(DATA_DIR)
+  const loader = new DataLoader({ storage })
 
-  const requests = [
-    client.download('channels.json'),
-    client.download('feeds.json'),
-    client.download('countries.json'),
-    client.download('regions.json'),
-    client.download('subdivisions.json')
-  ]
-
-  await Promise.all(requests)
+  await Promise.all([
+    loader.download('blocklist.json'),
+    loader.download('categories.json'),
+    loader.download('channels.json'),
+    loader.download('countries.json'),
+    loader.download('languages.json'),
+    loader.download('regions.json'),
+    loader.download('subdivisions.json'),
+    loader.download('feeds.json'),
+    loader.download('timezones.json'),
+    loader.download('guides.json'),
+    loader.download('streams.json')
+  ])
 }
 
 main()

scripts/commands/api/load.ts

@@ -4,13 +4,17 @@ import { ChannelsParser, XML } from '../../core'
 import { Channel, Feed } from '../../models'
 import { DATA_DIR } from '../../constants'
 import nodeCleanup from 'node-cleanup'
-import epgGrabber from 'epg-grabber'
 import { Command } from 'commander'
 import readline from 'readline'
-import Fuse from 'fuse.js'
+import sjs from '@freearhey/search-js'
+import { DataProcessor, DataLoader } from '../../core'
+import type { DataLoaderData } from '../../types/dataLoader'
+import type { DataProcessorData } from '../../types/dataProcessor'
+import epgGrabber from 'epg-grabber'
+import { ChannelSearchableData } from '../../types/channel'
 
 type ChoiceValue = { type: string; value?: Feed | Channel }
-type Choice = { name: string; short?: string; value: ChoiceValue }
+type Choice = { name: string; short?: string; value: ChoiceValue; default?: boolean }
 
 if (process.platform === 'win32') {
   readline
@@ -42,31 +46,48 @@ export default async function main(filepath: string) {
     throw new Error(`File "${filepath}" does not exists`)
   }
 
+  logger.info('loading data from api...')
+  const processor = new DataProcessor()
+  const dataStorage = new Storage(DATA_DIR)
+  const loader = new DataLoader({ storage: dataStorage })
+  const data: DataLoaderData = await loader.load()
+  const { feedsGroupedByChannelId, channels, channelsKeyById }: DataProcessorData =
+    processor.process(data)
+
+  logger.info('loading channels...')
   const parser = new ChannelsParser({ storage })
   parsedChannels = await parser.parse(filepath)
-
-  const dataStorage = new Storage(DATA_DIR)
-  const channelsData = await dataStorage.json('channels.json')
-  const channels = new Collection(channelsData).map(data => new Channel(data))
-  const feedsData = await dataStorage.json('feeds.json')
-  const feeds = new Collection(feedsData).map(data => new Feed(data))
-  const feedsGroupedByChannelId = feeds.groupBy((feed: Feed) => feed.channelId)
-
-  const searchIndex: Fuse<Channel> = new Fuse(channels.all(), {
-    keys: ['name', 'alt_names'],
-    threshold: 0.4
+  const parsedChannelsWithoutId = parsedChannels.filter(
+    (channel: epgGrabber.Channel) => !channel.xmltv_id
+  )
+
+  logger.info(
+    `found ${parsedChannels.count()} channels (including ${parsedChannelsWithoutId.count()} without ID)`
+  )
+
+  logger.info('creating search index...')
+  const items = channels.map((channel: Channel) => channel.getSearchable()).all()
+  const searchIndex = sjs.createIndex(items, {
+    searchable: ['name', 'altNames', 'guideNames', 'streamNames', 'feedFullNames']
   })
 
-  for (const channel of parsedChannels.all()) {
-    if (channel.xmltv_id) continue
+  logger.info('starting...\n')
+
+  for (const parsedChannel of parsedChannelsWithoutId.all()) {
     try {
-      channel.xmltv_id = await selectChannel(channel, searchIndex, feedsGroupedByChannelId)
-    } catch {
+      parsedChannel.xmltv_id = await selectChannel(
+        parsedChannel,
+        searchIndex,
+        feedsGroupedByChannelId,
+        channelsKeyById
+      )
+    } catch (err) {
+      logger.info(err.message)
       break
     }
   }
 
-  parsedChannels.forEach((channel: epgGrabber.Channel) => {
+  parsedChannelsWithoutId.forEach((channel: epgGrabber.Channel) => {
     if (channel.xmltv_id === '-') {
       channel.xmltv_id = ''
     }
@@ -75,12 +96,14 @@ export default async function main(filepath: string) {
 
 async function selectChannel(
   channel: epgGrabber.Channel,
-  searchIndex: Fuse<Channel>,
-  feedsGroupedByChannelId: Dictionary
+  searchIndex,
+  feedsGroupedByChannelId: Dictionary,
+  channelsKeyById: Dictionary
 ): Promise<string> {
+  const query = escapeRegex(channel.name)
   const similarChannels = searchIndex
-    .search(channel.name)
-    .map((result: { item: Channel }) => result.item)
+    .search(query)
+    .map((item: ChannelSearchableData) => channelsKeyById.get(item.id))
 
   const selected: ChoiceValue = await select({
     message: `Select channel ID for "${channel.name}" (${channel.site_id}):`,
@@ -93,13 +116,16 @@ async function selectChannel(
       return '-'
     case 'type': {
       const typedChannelId = await input({ message: '  Channel ID:' })
-      const typedFeedId = await input({ message: '  Feed ID:', default: 'SD' })
-      return [typedChannelId, typedFeedId].join('@')
+      if (!typedChannelId) return ''
+      const selectedFeedId = await selectFeed(typedChannelId, feedsGroupedByChannelId)
+      if (selectedFeedId === '-') return typedChannelId
+      return [typedChannelId, selectedFeedId].join('@')
     }
     case 'channel': {
       const selectedChannel = selected.value
       if (!selectedChannel) return ''
       const selectedFeedId = await selectFeed(selectedChannel.id, feedsGroupedByChannelId)
+      if (selectedFeedId === '-') return selectedChannel.id
       return [selectedChannel.id, selectedFeedId].join('@')
     }
   }
@@ -108,18 +134,22 @@ async function selectChannel(
 }
 
 async function selectFeed(channelId: string, feedsGroupedByChannelId: Dictionary): Promise<string> {
-  const channelFeeds = feedsGroupedByChannelId.get(channelId) || []
-  if (channelFeeds.length <= 1) return ''
+  const channelFeeds = feedsGroupedByChannelId.has(channelId)
+    ? new Collection(feedsGroupedByChannelId.get(channelId))
+    : new Collection()
+  const choices = getFeedChoises(channelFeeds)
 
   const selected: ChoiceValue = await select({
     message: `Select feed ID for "${channelId}":`,
-    choices: getFeedChoises(channelFeeds),
+    choices,
     pageSize: 10
   })
 
   switch (selected.type) {
+    case 'skip':
+      return '-'
     case 'type':
-      return await input({ message: '  Feed ID:' })
+      return await input({ message: '  Feed ID:', default: 'SD' })
     case 'feed':
       const selectedFeed = selected.value
       if (!selectedFeed) return ''
@@ -133,7 +163,7 @@ function getChannelChoises(channels: Collection): Choice[] {
   const choises: Choice[] = []
 
   channels.forEach((channel: Channel) => {
-    const names = [channel.name, ...channel.altNames.all()].join(', ')
+    const names = new Collection([channel.name, ...channel.getAltNames().all()]).uniq().join(', ')
 
     choises.push({
       value: {
@@ -163,12 +193,14 @@ function getFeedChoises(feeds: Collection): Choice[] {
         type: 'feed',
         value: feed
       },
+      default: feed.isMain,
       name,
       short: feed.id
     })
   })
 
   choises.push({ name: 'Type...', value: { type: 'type' } })
+  choises.push({ name: 'Skip', value: { type: 'skip' } })
 
   return choises
 }
@@ -179,3 +211,7 @@ function save(filepath: string) {
   storage.saveSync(filepath, xml.toString())
   logger.info(`\nFile '${filepath}' successfully saved`)
 }
+
+function escapeRegex(string: string) {
+  return string.replace(/[/\-\\^$*+?.()|[\]{}]/g, '\\$&')
+}

scripts/commands/channels/edit.ts

@@ -14,7 +14,7 @@ program
     )
   )
   .addOption(new Option('-o, --output <path>', 'Path to output file').default('guide.xml'))
-  .addOption(new Option('-l, --lang <code>', 'Filter channels by language (ISO 639-1 code)'))
+  .addOption(new Option('-l, --lang <codes>', 'Filter channels by languages (ISO 639-1 codes)'))
   .addOption(
     new Option('-t, --timeout <milliseconds>', 'Override the default timeout for each request').env(
       'TIMEOUT'
@@ -90,7 +90,11 @@ async function main() {
     parsedChannels = parsedChannels.concat(await parser.parse(filepath))
   }
   if (options.lang) {
-    parsedChannels = parsedChannels.filter((channel: Channel) => channel.lang === options.lang)
+    parsedChannels = parsedChannels.filter((channel: Channel) => {
+      if (!options.lang || !channel.lang) return true
+
+      return options.lang.includes(channel.lang)
+    })
   }
   logger.info(`  found ${parsedChannels.count()} channel(s)`)