feat: browse selected home sources with bounded recommendations

app/src/main/java/com/lagradost/cloudstream3/ui/APIRepository.kt

@@ -154,6 +154,7 @@ class APIRepository(val api: MainAPI) {
     }
 
     suspend fun getMainPage(page: Int, nameIndex: Int? = null): Resource<List<HomePageResponse?>> {
+        if (api is com.lagradost.cloudstream3.ui.home.CombinedHomeApi) return api.getPages(page, nameIndex)
         return safeApiCall {
             withTimeout(getTimeout(api.getMainPageTimeoutMs)) {
                 api.lastHomepageRequest = unixTimeMS

app/src/main/java/com/lagradost/cloudstream3/ui/home/CombinedHomeApi.kt

@@ -0,0 +1,83 @@
+package com.lagradost.cloudstream3.ui.home
+
+import com.lagradost.cloudstream3.HomePageResponse
+import com.lagradost.cloudstream3.MainAPI
+import com.lagradost.cloudstream3.MainPageData
+import com.lagradost.cloudstream3.MainPageRequest
+import com.lagradost.cloudstream3.newHomePageResponse
+import com.lagradost.cloudstream3.mvvm.Resource
+import kotlinx.coroutines.CancellationException
+import kotlinx.coroutines.async
+import kotlinx.coroutines.awaitAll
+import kotlinx.coroutines.coroutineScope
+import kotlinx.coroutines.delay
+import kotlinx.coroutines.sync.Mutex
+import kotlinx.coroutines.sync.Semaphore
+import kotlinx.coroutines.sync.withLock
+import kotlinx.coroutines.sync.withPermit
+import kotlinx.coroutines.withTimeoutOrNull
+import java.util.concurrent.ConcurrentHashMap
+
+internal class CombinedHomeApi(providers: List<MainAPI>) : MainAPI() {
+    companion object { const val NAME = "Selected sources" }
+    private data class Page(val provider: MainAPI, val data: MainPageData)
+    private val pages = providers.flatMap { provider -> provider.mainPage.map { Page(provider, it) } }
+    private val rowOwners = ConcurrentHashMap<String, Int>()
+    private val limits = Semaphore(3)
+    var onPartial: ((List<HomePageResponse>) -> Unit)? = null
+    private val providerLocks = providers.associateWith { Mutex() }
+    override var name = NAME
+    override var mainUrl = ""
+    override val hasMainPage = true
+    override val mainPage = pages.mapIndexed { index, page ->
+        MainPageData("${page.provider.name} · ${page.data.name}", index.toString(), page.data.horizontalImages)
+    }
+
+    fun pageIndexForRow(name: String): Int? = rowOwners[name]
+
+    override suspend fun getMainPage(page: Int, request: MainPageRequest): HomePageResponse {
+        val index = request.data.toIntOrNull() ?: return newHomePageResponse(emptyList(), false)
+        val owner = pages.getOrNull(index) ?: return newHomePageResponse(emptyList(), false)
+        return providerLocks.getValue(owner.provider).withLock {
+            limits.withPermit {
+                try {
+                    val response = withTimeoutOrNull(20_000) {
+                        val api = owner.provider
+                        val pause = api.lastHomepageRequest +
+                            maxOf(api.sequentialMainPageScrollDelay, if (api.sequentialMainPage) api.sequentialMainPageDelay else 0) -
+                            com.lagradost.cloudstream3.APIHolder.unixTimeMS
+                        if (pause > 0) delay(pause)
+                        api.lastHomepageRequest = com.lagradost.cloudstream3.APIHolder.unixTimeMS
+                        api.getMainPage(page, MainPageRequest(owner.data.name, owner.data.data, owner.data.horizontalImages))
+                    } ?: return@withPermit newHomePageResponse(emptyList(), false)
+                    newHomePageResponse(response.items.map { row ->
+                        val rowName = if (row.name == owner.data.name) "${owner.provider.name} · ${row.name}"
+                            else "${owner.provider.name} · ${owner.data.name}: ${row.name}"
+                        rowOwners[rowName] = index
+                        row.copy(name = rowName)
+                    }, response.hasNext)
+                } catch (cancelled: CancellationException) { throw cancelled }
+                catch (_: Exception) { newHomePageResponse(emptyList(), false) }
+            }
+        }
+    }
+
+    suspend fun getPages(page: Int, index: Int?): Resource<List<HomePageResponse?>> = coroutineScope {
+        val results = ConcurrentHashMap<Int, HomePageResponse>()
+        withTimeoutOrNull(35_000) {
+            (index?.let { listOf(it) } ?: pages.indices.toList()).map { position ->
+                async {
+                    mainPage.getOrNull(position)?.let { data ->
+                        val result = getMainPage(page, MainPageRequest(data.name, data.data, data.horizontalImages))
+                        if (result.items.isNotEmpty()) synchronized(results) {
+                            results[position] = result
+                            onPartial?.invoke(results.toSortedMap().values.toList())
+                        }
+                    }
+                }
+            }.awaitAll()
+        }
+        if (results.isEmpty()) Resource.Failure(false, "No selected source returned a catalogue")
+        else Resource.Success(results.toSortedMap().values.toList())
+    }
+}

app/src/main/java/com/lagradost/cloudstream3/ui/home/HomeFragment.kt

@@ -399,6 +399,10 @@ class HomeFragment : BaseFragment<FragmentHomeBinding>(
             )
 
             builder.setContentView(binding.root)
+            binding.multipleSourcesBtt.setOnClickListener {
+                builder.dismissSafe()
+                MultipleHomeSources.show(this, selectedApiName, callback)
+            }
             builder.show()
             builder.let { dialog ->
                 val isMultiLang = getApiProviderLangSettings().let { set ->

app/src/main/java/com/lagradost/cloudstream3/ui/home/HomeViewModel.kt

@@ -53,6 +53,13 @@ import com.lagradost.cloudstream3.utils.DataStoreHelper.getViewPos
 import com.lagradost.cloudstream3.utils.downloader.DownloadObjects
 import kotlinx.coroutines.Dispatchers
 import kotlinx.coroutines.Job
+import kotlinx.coroutines.async
+import kotlinx.coroutines.awaitAll
+import kotlinx.coroutines.coroutineScope
+import kotlinx.coroutines.sync.Semaphore
+import kotlinx.coroutines.sync.withPermit
+import kotlinx.coroutines.withTimeoutOrNull
+import java.util.concurrent.ConcurrentHashMap
 import kotlinx.coroutines.withContext
 import java.util.EnumSet
 import java.util.concurrent.CopyOnWriteArrayList
@@ -244,7 +251,10 @@ class HomeViewModel : ViewModel() {
                 }
 
                 val nextPage = current.currentPage + 1
-                val next = getMainPage(nextPage, mainPage.indexOfFirst { it.name == name })
+                val pageIndex = (api as? CombinedHomeApi)?.pageIndexForRow(name)
+                    ?: mainPage.indexOfFirst { it.name == name }
+                if (pageIndex < 0) { lock -= name; return null }
+                val next = getMainPage(nextPage, pageIndex)
                 if (next is Resource.Success) {
                     next.value.filterNotNull().forEach { main ->
                         main.items.forEach { newList ->
@@ -301,22 +311,45 @@ class HomeViewModel : ViewModel() {
             }
         }
 
-        val add = addItems.amap { searchResponse ->
-            repo?.load(searchResponse.url)
-        }.mapNotNull { if (it != null && it is Resource.Success) it.value else null }
+        val completed = ConcurrentHashMap<Int, LoadResponse>()
+        val permits = Semaphore(2)
+        withTimeoutOrNull(18_000) {
+            coroutineScope {
+                addItems.mapIndexed { index, searchResponse -> async {
+                    permits.withPermit {
+                        val response = withTimeoutOrNull(8_000) {
+                            getApiFromNameNull(searchResponse.apiName)?.let { APIRepository(it).load(searchResponse.url) }
+                        }
+                        if (response is Resource.Success) completed[index] = response.value
+                    }
+                } }.awaitAll()
+            }
+        }
+        val add = completed.toSortedMap().values.toList()
         current.addAll(add)
         return add.size
     }
 
     private var addJob: Job? = null
     fun loadMoreHomeScrollResponses() {
+        addJob?.cancel()
         addJob = ioSafe {
             updatePreviewResponses(previewResponses, previewResponsesAdded, currentShuffledList, 1)
             _preview.postValue(Resource.Success((previewResponsesAdded.size < currentShuffledList.size) to previewResponses))
         }
     }
 
     private fun load(api: MainAPI): Job = ioSafe {
+        if (api is CombinedHomeApi) api.onPartial = { responses ->
+            if (repo?.api === api) {
+                val partial = linkedMapOf<String, ExpandableHomepageList>()
+                responses.forEach { response -> response.items.forEach { row ->
+                    val filtered = context?.filterHomePageListByFilmQuality(row) ?: row
+                    partial[row.name] = ExpandableHomepageList(filtered, 1, false)
+                } }
+                _page.postValue(Resource.Success(partial))
+            }
+        }
         repo = //if (api != null) {
             APIRepository(api)
         //} else {
@@ -357,6 +390,7 @@ class HomeViewModel : ViewModel() {
                         }
                     }
 
+                    _page.postValue(Resource.Success(expandable.toMap()))
                     val items = data.value.mapNotNull { it?.items }.flatten()
 
 
@@ -517,6 +551,13 @@ class HomeViewModel : ViewModel() {
             }
 
             val api = getApiFromNameNull(preferredApiName)
+            if (preferredApiName == CombinedHomeApi.NAME) {
+                val providers = context?.let(MultipleHomeSources::selected).orEmpty()
+                if (fromUI) DataStoreHelper.currentHomePage = CombinedHomeApi.NAME
+                loadAndCancel(if (providers.isEmpty()) noneApi else CombinedHomeApi(providers))
+                reloadAccount()
+                return@ioSafe
+            }
             if (preferredApiName == noneApi.name) {
                 // just set to random
                 if (fromUI) DataStoreHelper.currentHomePage = noneApi.name
@@ -549,4 +590,4 @@ class HomeViewModel : ViewModel() {
             }
             reloadAccount()
         }
-}
+}

app/src/main/java/com/lagradost/cloudstream3/ui/home/MultipleHomeSources.kt

@@ -0,0 +1,43 @@
+package com.lagradost.cloudstream3.ui.home
+
+import android.content.Context
+import androidx.appcompat.app.AlertDialog
+import androidx.core.content.edit
+import androidx.preference.PreferenceManager
+import com.lagradost.cloudstream3.MainAPI
+import com.lagradost.cloudstream3.R
+import com.lagradost.cloudstream3.ui.APIRepository
+import com.lagradost.cloudstream3.utils.AppContextUtils.filterProviderByPreferredMedia
+
+internal object MultipleHomeSources {
+    fun active(context: Context): List<MainAPI> {
+        val current = com.lagradost.cloudstream3.utils.DataStoreHelper.currentHomePage
+        return if (current == CombinedHomeApi.NAME) selected(context)
+        else listOfNotNull(com.lagradost.cloudstream3.APIHolder.getApiFromNameNull(current)).filter { it.hasMainPage }
+    }
+    fun selected(context: Context): List<MainAPI> {
+        val prefs = PreferenceManager.getDefaultSharedPreferences(context)
+        val names = prefs.getStringSet("home_selected_sources", emptySet()).orEmpty()
+        return context.filterProviderByPreferredMedia().filter {
+            it.hasMainPage && (prefs.getBoolean("home_all_sources", false) || it.name in names)
+        }.distinctBy { it.name }
+    }
+
+    fun show(context: Context, current: String?, callback: (String) -> Unit) {
+        val providers = context.filterProviderByPreferredMedia().filter { it.hasMainPage }.distinctBy { it.name }
+        val prefs = PreferenceManager.getDefaultSharedPreferences(context)
+        val saved = prefs.getStringSet("home_selected_sources", current?.let { setOf(it) } ?: emptySet()).orEmpty()
+        val chosen = if (prefs.getBoolean("home_all_sources", false)) providers.map { it.name }.toMutableSet() else saved.toMutableSet()
+        val names = providers.map { it.name }.toTypedArray()
+        fun apply(all: Boolean) {
+            prefs.edit { putBoolean("home_all_sources", all); putStringSet("home_selected_sources", chosen.toSet()) }
+            callback(if (chosen.isEmpty()) APIRepository.noneApi.name else CombinedHomeApi.NAME)
+        }
+        AlertDialog.Builder(context).setTitle(R.string.home_multiple_sources)
+            .setMultiChoiceItems(names, BooleanArray(names.size) { names[it] in chosen }) { _, index, checked ->
+                if (checked) chosen.add(names[index]) else chosen.remove(names[index])
+            }.setPositiveButton(R.string.sort_apply) { _, _ -> apply(false) }
+            .setNeutralButton(R.string.home_all_sources) { _, _ -> chosen.addAll(names); apply(true) }
+            .setNegativeButton(android.R.string.cancel, null).show()
+    }
+}

app/src/main/java/com/lagradost/cloudstream3/ui/result/ProviderRecommendations.kt

@@ -0,0 +1,64 @@
+package com.lagradost.cloudstream3.ui.result
+
+import com.lagradost.cloudstream3.LoadResponse
+import com.lagradost.cloudstream3.MainAPI
+import com.lagradost.cloudstream3.ProviderType
+import com.lagradost.cloudstream3.SearchResponse
+import com.lagradost.cloudstream3.mvvm.Resource
+import com.lagradost.cloudstream3.ui.APIRepository
+import kotlinx.coroutines.async
+import kotlinx.coroutines.awaitAll
+import kotlinx.coroutines.coroutineScope
+import kotlinx.coroutines.ensureActive
+import kotlinx.coroutines.currentCoroutineContext
+import kotlinx.coroutines.sync.Semaphore
+import kotlinx.coroutines.sync.withPermit
+import kotlinx.coroutines.withTimeoutOrNull
+import java.text.Normalizer
+
+internal object ProviderRecommendations {
+    private fun title(value: String) = Normalizer.normalize(value, Normalizer.Form.NFD)
+        .replace("\\p{M}+".toRegex(), "").lowercase().replace("[^\\p{L}\\p{N}]".toRegex(), "")
+    suspend fun collect(item: LoadResponse, providers: List<MainAPI>, limit: Int, emit: (List<SearchResponse>) -> Unit) = coroutineScope {
+        val allowed = providers.map { it.name }.toSet()
+        val results = linkedMapOf<String, SearchResponse>()
+        val gate = Any()
+        fun add(items: List<SearchResponse>) = synchronized(gate) {
+            items.filter { (allowed.isEmpty() || it.apiName in allowed) && title(it.name) != title(item.name) }.forEach {
+                if (results.size < limit) results.putIfAbsent(title(it.name) + ":" + it.type, it)
+            }
+            emit(results.values.toList())
+        }
+        add(item.recommendations.orEmpty())
+        val permits = Semaphore(2)
+        withTimeoutOrNull(18_000) {
+            providers.map { provider -> async {
+                permits.withPermit {
+                    if (synchronized(gate) { results.size >= limit }) return@withPermit
+                    withTimeoutOrNull(8_000) {
+                        val repo = APIRepository(provider)
+                        if (provider.name != item.apiName && provider.providerType == ProviderType.DirectProvider) {
+                            val search = repo.search(item.name, 1)
+                            currentCoroutineContext().ensureActive()
+                            if (search is Resource.Success) {
+                                val match = search.value.items.firstOrNull { title(it.name) == title(item.name) }
+                                if (match != null) {
+                                    val loaded = repo.load(match.url)
+                                    currentCoroutineContext().ensureActive()
+                                    if (loaded is Resource.Success && (item.year == null || loaded.value.year == null || item.year == loaded.value.year)) {
+                                        add(loaded.value.recommendations.orEmpty())
+                                    }
+                                }
+                            }
+                        }
+                        if (synchronized(gate) { results.size < limit }) {
+                            val home = repo.getMainPage(1, 0)
+                            currentCoroutineContext().ensureActive()
+                            if (home is Resource.Success) add(home.value.filterNotNull().flatMap { it.items }.flatMap { it.list }.filter { it.type == item.type })
+                        }
+                    }
+                }
+            } }.awaitAll()
+        }
+    }
+}

app/src/main/java/com/lagradost/cloudstream3/ui/result/ResultFragmentPhone.kt

@@ -1449,13 +1449,13 @@ open class ResultFragmentPhone : BaseFragment<FragmentResultSwipeBinding>(
 
     private fun setRecommendations(rec: List<SearchResponse>?, validApiName: String?) {
         val isInvalid = rec.isNullOrEmpty()
-        val matchAgainst = validApiName ?: rec?.firstOrNull()?.apiName
+        val matchAgainst = validApiName
 
         recommendationBinding?.apply {
             root.isGone = isInvalid
             root.post {
                 rec?.let { list ->
-                    (resultRecommendationsList.adapter as? SearchAdapter)?.submitList(list.filter { it.apiName == matchAgainst })
+                    (resultRecommendationsList.adapter as? SearchAdapter)?.submitList(list.filter { matchAgainst == null || it.apiName == matchAgainst }.take(12))
                 }
             }
         }
@@ -1483,14 +1483,14 @@ open class ResultFragmentPhone : BaseFragment<FragmentResultSwipeBinding>(
                 // very dirty selection
                 recommendationBinding?.resultRecommendationsFilterButton?.apply {
                     isVisible = apiNames.size > 1
-                    text = matchAgainst
+                    text = matchAgainst ?: getString(R.string.recommendations_all_sources)
                     setOnClickListener { _ ->
                         activity?.showBottomDialog(
-                            apiNames,
-                            apiNames.indexOf(matchAgainst),
+                            listOf(getString(R.string.recommendations_all_sources)) + apiNames,
+                            if (matchAgainst == null) 0 else apiNames.indexOf(matchAgainst) + 1,
                             getString(R.string.home_change_provider_img_des), false, {}
                         ) {
-                            setRecommendations(rec, apiNames[it])
+                            setRecommendations(rec, if (it == 0) null else apiNames[it - 1])
                         }
                     }
                 }

app/src/main/java/com/lagradost/cloudstream3/ui/result/ResultFragmentTv.kt

@@ -155,19 +155,19 @@ class ResultFragmentTv : BaseFragment<FragmentResultTvBinding>(
         binding?.apply {
             resultRecommendationsList.isGone = isInvalid
             resultRecommendationsHolder.isGone = isInvalid
-            val matchAgainst = validApiName ?: rec?.firstOrNull()?.apiName
-            (resultRecommendationsList.adapter as? SearchAdapter)?.submitList(rec?.filter { it.apiName == matchAgainst }
+            val matchAgainst = validApiName?.takeUnless { it == "__all_selected_sources__" }
+            (resultRecommendationsList.adapter as? SearchAdapter)?.submitList(rec?.filter { matchAgainst == null || it.apiName == matchAgainst }?.take(12)
                 ?: emptyList())
 
             rec?.map { it.apiName }?.distinct()?.let { apiNames ->
                 // very dirty selection
                 resultRecommendationsFilterSelection.isVisible = apiNames.size > 1
-                resultRecommendationsFilterSelection.update(apiNames.map {
+                resultRecommendationsFilterSelection.update(listOf(txt(R.string.recommendations_all_sources) to "__all_selected_sources__") + apiNames.map {
                     txt(
                         it
                     ) to it
                 })
-                resultRecommendationsFilterSelection.select(apiNames.indexOf(matchAgainst))
+                resultRecommendationsFilterSelection.select(if (matchAgainst == null) 0 else apiNames.indexOf(matchAgainst) + 1)
             } ?: run {
                 resultRecommendationsFilterSelection.isVisible = false
             }