diff --git a/composeApp/src/androidFull/kotlin/com/nuvio/app/features/plugins/PluginPlatform.android.kt b/composeApp/src/androidFull/kotlin/com/nuvio/app/features/plugins/PluginPlatform.android.kt index 525b6c6f..96af5b11 100644 --- a/composeApp/src/androidFull/kotlin/com/nuvio/app/features/plugins/PluginPlatform.android.kt +++ b/composeApp/src/androidFull/kotlin/com/nuvio/app/features/plugins/PluginPlatform.android.kt @@ -32,6 +32,23 @@ internal object PluginStorage { ?.putString("settings_${scraperId}", payload) ?.apply() } + + fun loadCfSession(host: String): String? = + preferences?.getString("cf_session_${host}", null) + + fun saveCfSession(host: String, payload: String) { + preferences + ?.edit() + ?.putString("cf_session_${host}", payload) + ?.apply() + } + + fun removeCfSession(host: String) { + preferences + ?.edit() + ?.remove("cf_session_${host}") + ?.apply() + } } internal fun currentPluginPlatform(): String = "android" diff --git a/composeApp/src/androidFull/kotlin/com/nuvio/app/features/plugins/runtime/network/CloudflareKiller.android.kt b/composeApp/src/androidFull/kotlin/com/nuvio/app/features/plugins/runtime/network/CloudflareKiller.android.kt new file mode 100644 index 00000000..c8098aaf --- /dev/null +++ b/composeApp/src/androidFull/kotlin/com/nuvio/app/features/plugins/runtime/network/CloudflareKiller.android.kt @@ -0,0 +1,422 @@ +package com.nuvio.app.features.plugins.runtime.network + +import android.annotation.SuppressLint +import android.content.Context +import android.net.Uri +import android.net.http.SslError +import android.webkit.CookieManager +import android.webkit.SslErrorHandler +import android.webkit.WebResourceRequest +import android.webkit.WebResourceResponse +import android.webkit.WebView +import android.webkit.WebViewClient +import co.touchlab.kermit.Logger +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.NonCancellable +import kotlinx.coroutines.delay +import kotlinx.coroutines.withContext +import kotlinx.coroutines.withTimeoutOrNull +import kotlin.coroutines.resume +import kotlin.coroutines.suspendCoroutine + +internal fun platformWebViewSolverImpl(): WebViewSolver = AndroidWebViewSolver + +internal object AndroidWebViewSolver : WebViewSolver { + private val log = Logger.withTag("AndroidWebViewSolver") + + private const val PAGE_STATE_JS = """ + (function() { + try { + if (document.readyState !== 'interactive' && document.readyState !== 'complete') return 'wait'; + var title = (document.title || '').toLowerCase(); + if (title.indexOf('attention required') !== -1 || title.indexOf('access denied') !== -1) return 'blocked'; + if (title.indexOf('just a moment') !== -1) return 'wait'; + if (document.querySelector('#challenge-running, #challenge-stage, #cf-challenge-running, .cf-browser-verification, #turnstile-wrapper, #cf-please-wait, script[src*="challenge-platform"]')) return 'wait'; + if (!document.documentElement || !document.body) return 'wait'; + var contentType = (document.contentType || '').toLowerCase(); + if ((contentType.indexOf('json') !== -1 || contentType.indexOf('text/plain') !== -1) && (document.body.innerText || '').length > 0) return 'ok'; + var html = document.documentElement.outerHTML || ''; + if (html.length < 64) return 'wait'; + return 'ok'; + } catch (e) { + return 'wait'; + } + })() + """ + + private const val PAGE_BODY_JS = """ + (function() { + try { + var contentType = (document.contentType || '').toLowerCase(); + if (contentType.indexOf('json') !== -1 || contentType.indexOf('text/plain') !== -1) { + return document.body ? (document.body.innerText || '') : ''; + } + return document.documentElement ? (document.documentElement.outerHTML || '') : ''; + } catch (e) { + return ''; + } + })() + """ + + private const val PAGE_CONTENT_TYPE_JS = """ + (function() { + try { + return document.contentType || 'text/html'; + } catch (e) { + return 'text/html'; + } + })() + """ + + @Volatile + private var appContext: Context? = null + + private val blockedPathSuffixes = listOf( + ".jpg", + ".jpeg", + ".png", + ".webp", + ".mpg", + ".mpeg", + ".mp4", + ".webm", + ".gifv", + ".flv", + ".asf", + ".mov", + ".mng", + ".mkv", + ".ogg", + ".avi", + ".mp3", + ".wav", + ".woff2", + ".woff", + ".ttf", + ".css", + ".vtt", + ".srt", + ".ts", + ".gif", + ) + + fun initialize(context: Context) { + appContext = context.applicationContext + } + + @SuppressLint("SetJavaScriptEnabled") + override suspend fun solve( + url: String, + headers: Map, + forceFresh: Boolean, + timeoutMs: Long, + ): CfSolveResult? { + val context = appContext ?: run { + log.e { "AndroidWebViewSolver is not initialized" } + return null + } + + val host = extractHost(url) + val cookieManager = CookieManager.getInstance() + withContext(Dispatchers.Main) { + cookieManager.setAcceptCookie(true) + } + + if (!forceFresh) { + val existing = withContext(Dispatchers.Main) { + runCatching { cookieManager.getCookie(url) }.getOrNull() + } + if (existing?.contains("cf_clearance") == true) { + return CfSolveResult( + cookies = parseCookieString(existing), + userAgent = captureUserAgent(context), + ) + } + } + + runCatching { clearCookiesForUrl(url) } + + var webViewRef: WebView? = null + var webViewUserAgent = "" + val finalUrlRef = java.util.concurrent.atomic.AtomicReference(url) + + try { + withContext(Dispatchers.Main) { + val webView = WebView(context).apply { + settings.javaScriptEnabled = true + settings.domStorageEnabled = true + webViewUserAgent = settings.userAgentString + + CookieManager.getInstance().setAcceptThirdPartyCookies(this, true) + + webViewClient = object : WebViewClient() { + override fun onPageStarted( + view: WebView?, + pageUrl: String?, + favicon: android.graphics.Bitmap?, + ) { + if (pageUrl != null && !pageUrl.contains("challenges.cloudflare.com")) { + finalUrlRef.set(pageUrl) + } + super.onPageStarted(view, pageUrl, favicon) + } + + override fun onPageFinished(view: WebView?, pageUrl: String?) { + if (pageUrl != null && !pageUrl.contains("challenges.cloudflare.com")) { + finalUrlRef.set(pageUrl) + } + super.onPageFinished(view, pageUrl) + } + + override fun shouldInterceptRequest( + view: WebView, + request: WebResourceRequest, + ): WebResourceResponse? { + val requestUrl = request.url.toString() + + if (requestUrl.contains("recaptcha") || requestUrl.contains("/cdn-cgi/")) { + return super.shouldInterceptRequest(view, request) + } + + if (requestUrl.endsWith("/favicon.ico") || requestUrl.startsWith("wss://")) { + return WebResourceResponse("text/plain", "utf-8", null) + } + + val path = runCatching { Uri.parse(requestUrl).path.orEmpty() }.getOrDefault("") + if (blockedPathSuffixes.any { path.contains(it, ignoreCase = true) }) { + return WebResourceResponse("text/plain", "utf-8", null) + } + + return super.shouldInterceptRequest(view, request) + } + + @SuppressLint("WebViewClientOnReceivedSslError") + override fun onReceivedSslError( + view: WebView?, + handler: SslErrorHandler?, + error: SslError?, + ) { + handler?.proceed() + } + } + } + webViewRef = webView + webView.loadUrl(url, headers) + } + + val result = withTimeoutOrNull(timeoutMs) { + var solved: CfSolveResult? = null + while (solved == null) { + delay(100L) + val currentUrl = finalUrlRef.get() + val raw = runCatching { + cookieManager.getCookie(currentUrl) ?: cookieManager.getCookie(url) + }.getOrNull() + + if (raw?.contains("cf_clearance") == true) { + delay(500L) + val finalUrl = finalUrlRef.get() + val finalRaw = runCatching { + cookieManager.getCookie(finalUrl) ?: cookieManager.getCookie(url) + }.getOrNull() ?: raw + solved = CfSolveResult( + cookies = parseCookieString(finalRaw), + userAgent = webViewUserAgent, + redirectUrl = finalUrl.takeIf { it != url }, + ) + } + } + solved + } + + if (result == null) { + log.w { "CF solve timed out after ${timeoutMs}ms for $host" } + } + return result + } finally { + withContext(NonCancellable + Dispatchers.Main) { + webViewRef?.stopLoading() + webViewRef?.destroy() + webViewRef = null + } + } + } + + @SuppressLint("SetJavaScriptEnabled") + override suspend fun fetchRenderedPage( + url: String, + headers: Map, + timeoutMs: Long, + ): WebViewFetchResult? { + val context = appContext ?: run { + log.e { "AndroidWebViewSolver is not initialized" } + return null + } + + var webViewRef: WebView? = null + val finalUrlRef = java.util.concurrent.atomic.AtomicReference(url) + val statusRef = java.util.concurrent.atomic.AtomicInteger(200) + val statusTextRef = java.util.concurrent.atomic.AtomicReference("OK") + val responseHeadersRef = java.util.concurrent.atomic.AtomicReference>(emptyMap()) + + try { + withContext(Dispatchers.Main) { + val webView = WebView(context).apply { + settings.javaScriptEnabled = true + settings.domStorageEnabled = true + CookieManager.getInstance().setAcceptThirdPartyCookies(this, true) + + webViewClient = object : WebViewClient() { + override fun onPageStarted( + view: WebView?, + pageUrl: String?, + favicon: android.graphics.Bitmap?, + ) { + pageUrl?.let(finalUrlRef::set) + super.onPageStarted(view, pageUrl, favicon) + } + + override fun onPageFinished(view: WebView?, pageUrl: String?) { + pageUrl?.let(finalUrlRef::set) + super.onPageFinished(view, pageUrl) + } + + override fun onReceivedHttpError( + view: WebView?, + request: WebResourceRequest?, + errorResponse: WebResourceResponse?, + ) { + if (request?.isForMainFrame == true && errorResponse != null) { + statusRef.set(errorResponse.statusCode) + statusTextRef.set(errorResponse.reasonPhrase ?: "") + responseHeadersRef.set(errorResponse.responseHeaders.orEmpty()) + } + super.onReceivedHttpError(view, request, errorResponse) + } + + override fun shouldInterceptRequest( + view: WebView, + request: WebResourceRequest, + ): WebResourceResponse? { + val requestUrl = request.url.toString() + if (requestUrl.contains("recaptcha") || requestUrl.contains("/cdn-cgi/")) { + return super.shouldInterceptRequest(view, request) + } + if (requestUrl.endsWith("/favicon.ico") || requestUrl.startsWith("wss://")) { + return WebResourceResponse("text/plain", "utf-8", null) + } + return super.shouldInterceptRequest(view, request) + } + + @SuppressLint("WebViewClientOnReceivedSslError") + override fun onReceivedSslError( + view: WebView?, + handler: SslErrorHandler?, + error: SslError?, + ) { + handler?.proceed() + } + } + } + webViewRef = webView + webView.loadUrl(url, headers) + } + + val result = withTimeoutOrNull(timeoutMs) { + var rendered: WebViewFetchResult? = null + while (rendered == null) { + delay(250L) + val state = withContext(Dispatchers.Main) { + webViewRef?.evaluateJavascriptString(PAGE_STATE_JS) + } ?: "wait" + + if (state == "blocked") { + return@withTimeoutOrNull null + } + + if (state == "ok") { + val body = withContext(Dispatchers.Main) { + webViewRef?.evaluateJavascriptString(PAGE_BODY_JS) + }.orEmpty() + if (body.isNotBlank()) { + val contentType = withContext(Dispatchers.Main) { + webViewRef?.evaluateJavascriptString(PAGE_CONTENT_TYPE_JS) + }.orEmpty().ifBlank { "text/html" } + val headersWithContentType = responseHeadersRef.get().toMutableMap() + if (headersWithContentType.keys.none { it.equals("content-type", ignoreCase = true) }) { + headersWithContentType["content-type"] = contentType + } + rendered = WebViewFetchResult( + status = statusRef.get(), + statusText = statusTextRef.get(), + url = finalUrlRef.get(), + body = body, + headers = headersWithContentType, + ) + } + } + } + rendered + } + + if (result == null) { + log.w { "CF WebView fetch fallback timed out after ${timeoutMs}ms for $url" } + } + return result + } finally { + withContext(NonCancellable + Dispatchers.Main) { + webViewRef?.stopLoading() + webViewRef?.destroy() + webViewRef = null + } + } + } + + private suspend fun clearCookiesForUrl(url: String) = withContext(Dispatchers.Main) { + val cookieString = CookieManager.getInstance().getCookie(url).orEmpty() + if (cookieString.isBlank()) return@withContext + + val host = Uri.parse(url).host ?: return@withContext + val rootDomain = host.split(".").takeIf { it.size > 2 } + ?.takeLast(2) + ?.joinToString(separator = ".", prefix = ".") + val cookieNames = parseCookieString(cookieString).keys + + cookieNames.forEach { name -> + expireCookie(url, "$name=; Expires=Thu, 01 Jan 1970 00:00:00 GMT; Path=/; Domain=$host") + if (rootDomain != null) { + expireCookie(url, "$name=; Expires=Thu, 01 Jan 1970 00:00:00 GMT; Path=/; Domain=$rootDomain") + } + } + CookieManager.getInstance().flush() + } + + private suspend fun expireCookie(url: String, value: String) = suspendCoroutine { cont -> + CookieManager.getInstance().setCookie(url, value) { + cont.resume(Unit) + } + } + + private suspend fun captureUserAgent(context: Context): String = withContext(Dispatchers.Main) { + val webView = WebView(context) + val userAgent = webView.settings.userAgentString + webView.destroy() + userAgent + } + + private suspend fun WebView.evaluateJavascriptString(script: String): String = + suspendCoroutine { cont -> + evaluateJavascript(script) { raw -> + cont.resume(decodeJavascriptString(raw)) + } + } + + private fun decodeJavascriptString(raw: String?): String { + if (raw == null || raw == "null") return "" + return runCatching { + org.json.JSONArray("[$raw]").getString(0) + }.getOrElse { + raw.removeSurrounding("\"") + } + } +} diff --git a/composeApp/src/androidFull/kotlin/com/nuvio/app/features/plugins/runtime/network/CloudflareRuntimeInitializer.android.kt b/composeApp/src/androidFull/kotlin/com/nuvio/app/features/plugins/runtime/network/CloudflareRuntimeInitializer.android.kt new file mode 100644 index 00000000..f504edd3 --- /dev/null +++ b/composeApp/src/androidFull/kotlin/com/nuvio/app/features/plugins/runtime/network/CloudflareRuntimeInitializer.android.kt @@ -0,0 +1,9 @@ +package com.nuvio.app.features.plugins.runtime.network + +import android.content.Context + +internal object CloudflareRuntimeInitializer { + fun initialize(context: Context) { + AndroidWebViewSolver.initialize(context) + } +} diff --git a/composeApp/src/androidMain/kotlin/com/nuvio/app/MainActivity.kt b/composeApp/src/androidMain/kotlin/com/nuvio/app/MainActivity.kt index d13a2453..9f87ec12 100644 --- a/composeApp/src/androidMain/kotlin/com/nuvio/app/MainActivity.kt +++ b/composeApp/src/androidMain/kotlin/com/nuvio/app/MainActivity.kt @@ -37,6 +37,7 @@ import com.nuvio.app.features.player.PipRemoteActionReceiver import com.nuvio.app.features.p2p.P2pSettingsStorage import com.nuvio.app.features.p2p.P2pStreamingEngine import com.nuvio.app.features.plugins.PluginStorage +import com.nuvio.app.features.plugins.runtime.network.CloudflareRuntimeInitializer import com.nuvio.app.features.profiles.AvatarStorage import com.nuvio.app.features.profiles.ProfilePinCacheStorage import com.nuvio.app.features.profiles.ProfileStorage @@ -114,6 +115,7 @@ class MainActivity : AppCompatActivity() { StreamBadgeSettingsStorage.initialize(applicationContext) BingeGroupCacheStorage.initialize(applicationContext) PluginStorage.initialize(applicationContext) + CloudflareRuntimeInitializer.initialize(applicationContext) CollectionMobileSettingsStorage.initialize(applicationContext) CollectionStorage.initialize(applicationContext) DownloadsStorage.initialize(applicationContext) diff --git a/composeApp/src/androidPlaystore/kotlin/com/nuvio/app/features/plugins/runtime/network/CloudflareRuntimeInitializer.android.kt b/composeApp/src/androidPlaystore/kotlin/com/nuvio/app/features/plugins/runtime/network/CloudflareRuntimeInitializer.android.kt new file mode 100644 index 00000000..3718ff33 --- /dev/null +++ b/composeApp/src/androidPlaystore/kotlin/com/nuvio/app/features/plugins/runtime/network/CloudflareRuntimeInitializer.android.kt @@ -0,0 +1,7 @@ +package com.nuvio.app.features.plugins.runtime.network + +import android.content.Context + +internal object CloudflareRuntimeInitializer { + fun initialize(context: Context) = Unit +} diff --git a/composeApp/src/fullCommonMain/kotlin/com/nuvio/app/features/plugins/runtime/PluginRuntime.kt b/composeApp/src/fullCommonMain/kotlin/com/nuvio/app/features/plugins/runtime/PluginRuntime.kt index fc014ee7..25c49d9f 100644 --- a/composeApp/src/fullCommonMain/kotlin/com/nuvio/app/features/plugins/runtime/PluginRuntime.kt +++ b/composeApp/src/fullCommonMain/kotlin/com/nuvio/app/features/plugins/runtime/PluginRuntime.kt @@ -132,7 +132,7 @@ internal object PluginRuntime { val domBridge = DomBridge() val hostRegistry = HostApiRegistry().apply { addModule(HostFunctions(scraperId) { deferred.complete(it) }) - addModule(FetchBridge()) + addModule(FetchBridge(scraperId)) addModule(UrlBridge()) addModule(CryptoBridge()) addModule(WasmBridge()) diff --git a/composeApp/src/fullCommonMain/kotlin/com/nuvio/app/features/plugins/runtime/js/JsBindings.kt b/composeApp/src/fullCommonMain/kotlin/com/nuvio/app/features/plugins/runtime/js/JsBindings.kt index 5e239048..a3d3919c 100644 --- a/composeApp/src/fullCommonMain/kotlin/com/nuvio/app/features/plugins/runtime/js/JsBindings.kt +++ b/composeApp/src/fullCommonMain/kotlin/com/nuvio/app/features/plugins/runtime/js/JsBindings.kt @@ -47,7 +47,8 @@ internal object JsBindings { var headers = __normalize_fetch_headers(options.headers); var body = options.body || ''; var followRedirects = options.redirect !== 'manual'; - var result = __native_fetch(url, method, JSON.stringify(headers), body, followRedirects); + var useCfKiller = options.cfKiller === true; + var result = __native_fetch(url, method, JSON.stringify(headers), body, followRedirects, useCfKiller); var parsed = JSON.parse(result); return { ok: parsed.ok, diff --git a/composeApp/src/fullCommonMain/kotlin/com/nuvio/app/features/plugins/runtime/network/CloudflareKiller.kt b/composeApp/src/fullCommonMain/kotlin/com/nuvio/app/features/plugins/runtime/network/CloudflareKiller.kt new file mode 100644 index 00000000..126226b0 --- /dev/null +++ b/composeApp/src/fullCommonMain/kotlin/com/nuvio/app/features/plugins/runtime/network/CloudflareKiller.kt @@ -0,0 +1,151 @@ +package com.nuvio.app.features.plugins.runtime.network + +import kotlinx.atomicfu.locks.SynchronizedObject +import kotlinx.atomicfu.locks.synchronized +import kotlinx.coroutines.sync.Mutex +import kotlinx.serialization.json.Json +import kotlinx.serialization.json.buildJsonObject +import kotlinx.serialization.json.jsonObject +import kotlinx.serialization.json.jsonPrimitive +import kotlinx.serialization.json.put + +internal data class CfSolveResult( + val cookies: Map, + val userAgent: String, + val redirectUrl: String? = null, +) + +internal data class WebViewFetchResult( + val status: Int, + val statusText: String, + val url: String, + val body: String, + val headers: Map = emptyMap(), +) + +internal interface WebViewSolver { + suspend fun solve( + url: String, + headers: Map = emptyMap(), + forceFresh: Boolean = false, + timeoutMs: Long = 60_000L, + ): CfSolveResult? + + suspend fun fetchRenderedPage( + url: String, + headers: Map = emptyMap(), + timeoutMs: Long = 60_000L, + ): WebViewFetchResult? = null +} + +internal fun createPlatformWebViewSolver(): WebViewSolver = platformWebViewSolverImpl() + +internal object CfSessionCache { + private val sessions = mutableMapOf() + private val hostLocks = mutableMapOf() + private val lock = SynchronizedObject() + + fun getMutex(host: String): Mutex = synchronized(lock) { + hostLocks.getOrPut(host) { Mutex() } + } + + fun get(host: String, pluginId: String): CfSolveResult? = synchronized(lock) { + sessions[host]?.let { return it } + val persisted = runCatching { + com.nuvio.app.features.plugins.PluginStorage.loadCfSession(host) + }.getOrNull() + val parsed = persisted?.let(::jsonStringToCfSolveResult) ?: return null + sessions[host] = parsed + parsed + } + + fun put(host: String, pluginId: String, result: CfSolveResult) = synchronized(lock) { + sessions[host] = result + runCatching { + com.nuvio.app.features.plugins.PluginStorage.saveCfSession(host, result.toJsonString()) + } + } + + fun evict(host: String, pluginId: String) = synchronized(lock) { + sessions.remove(host) + runCatching { + com.nuvio.app.features.plugins.PluginStorage.removeCfSession(host) + } + } + + fun evictIfSame(host: String, result: CfSolveResult) = synchronized(lock) { + if (sessions[host] == result) { + sessions.remove(host) + runCatching { + com.nuvio.app.features.plugins.PluginStorage.removeCfSession(host) + } + } + } + + fun clear() = synchronized(lock) { + sessions.clear() + } +} + +private val COOKIE_DIRECTIVES = setOf( + "path", + "domain", + "expires", + "max-age", + "secure", + "httponly", + "samesite", + "priority", +) + +internal fun parseCookieString(raw: String): Map = + raw.split(";").mapNotNull { part -> + val eq = part.indexOf('=') + if (eq < 0) return@mapNotNull null + val name = part.substring(0, eq).trim() + val value = part.substring(eq + 1).trim() + if (name.isBlank() || value.isBlank() || name.lowercase() in COOKIE_DIRECTIVES) { + null + } else { + name to value + } + }.toMap() + +internal fun Map.toCookieHeader(): String = + entries.joinToString("; ") { (key, value) -> "$key=$value" } + +internal fun extractHost(url: String): String = runCatching { + val authority = url + .substringAfter("://", url) + .substringBefore("/") + .substringBefore("?") + .substringBefore("#") + .substringAfterLast("@") + if (authority.startsWith("[")) { + authority.substringBefore("]").removePrefix("[") + } else { + authority.substringBefore(":") + }.lowercase() +}.getOrDefault(url) + +private fun CfSolveResult.toJsonString(): String = buildJsonObject { + put("cookies", buildJsonObject { + cookies.forEach { (key, value) -> put(key, value) } + }) + put("userAgent", userAgent) + redirectUrl?.let { put("redirectUrl", it) } +}.toString() + +private fun jsonStringToCfSolveResult(raw: String): CfSolveResult? = runCatching { + val obj = Json.parseToJsonElement(raw).jsonObject + val cookies = obj["cookies"]?.jsonObject?.entries?.associate { (key, value) -> + key to value.jsonPrimitive.content + }.orEmpty() + val userAgent = obj["userAgent"]?.jsonPrimitive?.content.orEmpty() + val redirectUrl = obj["redirectUrl"]?.jsonPrimitive?.content + if (cookies.isEmpty() || userAgent.isBlank()) { + null + } else { + CfSolveResult(cookies = cookies, userAgent = userAgent, redirectUrl = redirectUrl) + } +}.getOrNull() diff --git a/composeApp/src/fullCommonMain/kotlin/com/nuvio/app/features/plugins/runtime/network/FetchBridge.kt b/composeApp/src/fullCommonMain/kotlin/com/nuvio/app/features/plugins/runtime/network/FetchBridge.kt index 22216c58..969af707 100644 --- a/composeApp/src/fullCommonMain/kotlin/com/nuvio/app/features/plugins/runtime/network/FetchBridge.kt +++ b/composeApp/src/fullCommonMain/kotlin/com/nuvio/app/features/plugins/runtime/network/FetchBridge.kt @@ -5,7 +5,9 @@ import com.dokar.quickjs.QuickJs import com.dokar.quickjs.binding.function import com.nuvio.app.features.addons.httpRequestRaw import com.nuvio.app.features.plugins.runtime.host.HostModule +import kotlinx.coroutines.delay import kotlinx.coroutines.runBlocking +import kotlinx.coroutines.sync.withLock import kotlinx.serialization.json.Json import kotlinx.serialization.json.JsonObject import kotlinx.serialization.json.JsonPrimitive @@ -14,10 +16,13 @@ import kotlinx.serialization.json.jsonPrimitive private const val MAX_FETCH_HEADER_VALUE_CHARS = 8 * 1024 private const val FETCH_TRUNCATION_SUFFIX = "\n...[truncated]" +private val CF_BLOCK_CODES = setOf(403, 503) +private val CF_SERVER_MARKERS = setOf("cloudflare", "cloudflare-nginx") -internal class FetchBridge : HostModule { +internal class FetchBridge(private val pluginId: String) : HostModule { private val log = Logger.withTag("PluginRuntime") private val json = Json { ignoreUnknownKeys = true } + private val cfSolver: WebViewSolver by lazy { createPlatformWebViewSolver() } override fun register(runtime: QuickJs) { runtime.function("__native_fetch") { args -> @@ -26,8 +31,9 @@ internal class FetchBridge : HostModule { val headersJson = args.getOrNull(2)?.toString() ?: "{}" val body = args.getOrNull(3)?.toString() ?: "" val followRedirects = args.getOrNull(4) as? Boolean ?: true + val useCfKiller = args.getOrNull(5) as? Boolean ?: false try { - performNativeFetch(url, method, headersJson, body, followRedirects) + performNativeFetch(url, method, headersJson, body, followRedirects, useCfKiller) } catch (t: Throwable) { log.e(t) { "Fetch bridge error for $method $url" } JsonObject( @@ -50,10 +56,25 @@ internal class FetchBridge : HostModule { headersJson: String, body: String, followRedirects: Boolean, + useCfKiller: Boolean, ): String { val headers = parseHeaders(headersJson).toMutableMap() - if (!headers.containsKey("User-Agent")) { - headers["User-Agent"] = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36" + if (headers.getIgnoreCase("User-Agent") == null) { + headers["User-Agent"] = "Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Mobile Safari/537.36" + } + + val host = extractHost(url) + var usedCachedSession: CfSolveResult? = null + if (useCfKiller) { + CfSessionCache.get(host, pluginId)?.let { cached -> + headers.setIgnoreCase("User-Agent", cached.userAgent) + headers.setIgnoreCase( + "Cookie", + mergeCookies(headers.getIgnoreCase("Cookie").orEmpty(), cached.cookies.toCookieHeader()), + ) + usedCachedSession = cached + log.d { "CF: using cached session for $host" } + } } val response = runBlocking { @@ -66,6 +87,110 @@ internal class FetchBridge : HostModule { ) } + if (useCfKiller && isCloudflareBlocked(response.status, response.headers)) { + log.i { "CF: blocked (${response.status}) at $url; launching WebView solver" } + val solveResult = runBlocking { + CfSessionCache.getMutex(host).withLock { + usedCachedSession?.let { CfSessionCache.evictIfSame(host, it) } + + CfSessionCache.get(host, pluginId) ?: cfSolver + .solve( + url = url, + headers = webViewHeaders(headers), + forceFresh = true, + ) + ?.also { solved -> + CfSessionCache.put(host, pluginId, solved) + } + } + } + + if (solveResult != null) { + solveResult.redirectUrl?.let { redirectUrl -> + val redirectHost = extractHost(redirectUrl) + if (redirectHost != host) { + CfSessionCache.put(redirectHost, pluginId, solveResult) + } + } + + val retryHeaders = parseHeaders(headersJson).toMutableMap() + retryHeaders.setIgnoreCase("User-Agent", solveResult.userAgent) + retryHeaders.setIgnoreCase( + "Cookie", + mergeCookies(retryHeaders.getIgnoreCase("Cookie").orEmpty(), solveResult.cookies.toCookieHeader()), + ) + + val retryUrl = solveResult.redirectUrl ?: url + var retryResponse = runBlocking { + httpRequestRaw( + method = method, + url = retryUrl, + headers = retryHeaders, + body = body, + followRedirects = followRedirects, + ) + } + + if (isCloudflareBlocked(retryResponse.status, retryResponse.headers)) { + runBlocking { delay(500L) } + retryResponse = runBlocking { + httpRequestRaw( + method = method, + url = retryUrl, + headers = retryHeaders, + body = body, + followRedirects = followRedirects, + ) + } + } + + if (isCloudflareBlocked(retryResponse.status, retryResponse.headers)) { + log.w { "CF: retry still blocked (${retryResponse.status}) at $retryUrl; trying WebView fetch fallback" } + val renderedResponse = runBlocking { + cfSolver.fetchRenderedPage( + url = retryUrl, + headers = webViewHeaders(retryHeaders), + ) + } + + if (renderedResponse != null && !isRenderedCloudflareBlocked(renderedResponse)) { + log.i { "CF: WebView fetch fallback success for ${renderedResponse.url}" } + return buildResponseJson(renderedResponse) + } + + log.w { "CF: WebView fetch fallback failed for $retryUrl; evicting session" } + CfSessionCache.evict(host, pluginId) + solveResult.redirectUrl?.let { CfSessionCache.evict(extractHost(it), pluginId) } + } else { + log.i { "CF: solve success (${retryResponse.status}) for $retryUrl" } + } + + return buildResponseJson(retryResponse) + } + + log.w { "CF: WebView solver timed out for $url" } + } + + return buildResponseJson(response) + } + + private fun buildResponseJson(response: com.nuvio.app.features.addons.RawHttpResponse): String { + val responseHeaders = response.headers.mapKeys { (key, _) -> key.lowercase() } + .mapValues { (_, value) -> truncateString(value, MAX_FETCH_HEADER_VALUE_CHARS) } + val result = JsonObject( + mapOf( + "ok" to JsonPrimitive(response.status in 200..299), + "status" to JsonPrimitive(response.status), + "statusText" to JsonPrimitive(response.statusText), + "url" to JsonPrimitive(response.url), + "body" to JsonPrimitive(response.body), + "headers" to JsonObject(responseHeaders.mapValues { JsonPrimitive(it.value) }), + ), + ) + return result.toString() + } + + private fun buildResponseJson(response: WebViewFetchResult): String { val responseHeaders = response.headers.mapKeys { (key, _) -> key.lowercase() } .mapValues { (_, value) -> truncateString(value, MAX_FETCH_HEADER_VALUE_CHARS) } val result = JsonObject( @@ -92,10 +217,49 @@ internal class FetchBridge : HostModule { }.getOrDefault(emptyMap()) } + private fun isCloudflareBlocked(status: Int, headers: Map): Boolean { + if (status !in CF_BLOCK_CODES) return false + val server = headers.getIgnoreCase("Server").orEmpty().lowercase() + val hasCfServer = CF_SERVER_MARKERS.any { marker -> server.contains(marker) } + val hasCfHeaders = headers.keys.any { key -> key.lowercase().startsWith("cf-") } + return hasCfServer || hasCfHeaders + } + + private fun isRenderedCloudflareBlocked(response: WebViewFetchResult): Boolean { + if (isCloudflareBlocked(response.status, response.headers)) return true + val body = response.body.lowercase() + return body.contains("cf-browser-verification") || + body.contains("challenge-platform") || + body.contains("cf-challenge-running") || + body.contains("turnstile-wrapper") || + body.contains("just a moment") + } + + private fun mergeCookies(existing: String, extra: String): String { + if (existing.isBlank()) return extra + if (extra.isBlank()) return existing + return (parseCookieString(existing) + parseCookieString(extra)).toCookieHeader() + } + + private fun webViewHeaders(headers: Map): Map = + headers.filterKeys { key -> + !key.equals("User-Agent", ignoreCase = true) && + !key.equals("Cookie", ignoreCase = true) && + !key.equals("X-Requested-With", ignoreCase = true) + } + private fun truncateString(value: String, maxChars: Int): String { if (value.length <= maxChars) return value val end = maxChars - FETCH_TRUNCATION_SUFFIX.length if (end <= 0) return FETCH_TRUNCATION_SUFFIX.take(maxChars) return value.substring(0, end) + FETCH_TRUNCATION_SUFFIX } + + private fun MutableMap.setIgnoreCase(name: String, value: String) { + keys.filter { it.equals(name, ignoreCase = true) }.forEach { remove(it) } + put(name, value) + } + + private fun Map.getIgnoreCase(name: String): String? = + entries.firstOrNull { (key, _) -> key.equals(name, ignoreCase = true) }?.value } diff --git a/composeApp/src/iosFull/kotlin/com/nuvio/app/features/plugins/PluginPlatform.ios.kt b/composeApp/src/iosFull/kotlin/com/nuvio/app/features/plugins/PluginPlatform.ios.kt index 1a8ddaac..ca6981fa 100644 --- a/composeApp/src/iosFull/kotlin/com/nuvio/app/features/plugins/PluginPlatform.ios.kt +++ b/composeApp/src/iosFull/kotlin/com/nuvio/app/features/plugins/PluginPlatform.ios.kt @@ -25,6 +25,20 @@ internal object PluginStorage { forKey = "settings_${scraperId}", ) } + + fun loadCfSession(host: String): String? = + NSUserDefaults.standardUserDefaults.stringForKey("cf_session_${host}") + + fun saveCfSession(host: String, payload: String) { + NSUserDefaults.standardUserDefaults.setObject( + payload, + forKey = "cf_session_${host}", + ) + } + + fun removeCfSession(host: String) { + NSUserDefaults.standardUserDefaults.removeObjectForKey("cf_session_${host}") + } } internal fun currentPluginPlatform(): String = "ios" diff --git a/composeApp/src/iosFull/kotlin/com/nuvio/app/features/plugins/runtime/network/CloudflareKiller.ios.kt b/composeApp/src/iosFull/kotlin/com/nuvio/app/features/plugins/runtime/network/CloudflareKiller.ios.kt new file mode 100644 index 00000000..7f208693 --- /dev/null +++ b/composeApp/src/iosFull/kotlin/com/nuvio/app/features/plugins/runtime/network/CloudflareKiller.ios.kt @@ -0,0 +1,370 @@ +package com.nuvio.app.features.plugins.runtime.network + +import co.touchlab.kermit.Logger +import kotlinx.cinterop.ExperimentalForeignApi +import kotlinx.cinterop.ObjCSignatureOverride +import kotlinx.cinterop.readValue +import kotlinx.coroutines.CompletableDeferred +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.NonCancellable +import kotlinx.coroutines.delay +import kotlinx.coroutines.withContext +import kotlinx.coroutines.withTimeoutOrNull +import kotlin.coroutines.resume +import kotlin.coroutines.suspendCoroutine +import platform.Foundation.* +import platform.WebKit.* +import platform.darwin.NSObject +import platform.Security.SecTrustRef + + +internal fun platformWebViewSolverImpl(): WebViewSolver = IosWebViewSolver + +@OptIn(ExperimentalForeignApi::class) +internal object IosWebViewSolver : WebViewSolver { + private val log = Logger.withTag("IosWebViewSolver") + private var cachedUserAgent: String? = null + + private const val PAGE_STATE_JS = """ + (function() { + try { + if (document.readyState !== 'interactive' && document.readyState !== 'complete') return 'wait'; + var title = (document.title || '').toLowerCase(); + if (title.indexOf('attention required') !== -1 || title.indexOf('access denied') !== -1) return 'blocked'; + if (title.indexOf('just a moment') !== -1) return 'wait'; + if (document.querySelector('#challenge-running, #challenge-stage, #cf-challenge-running, .cf-browser-verification, #turnstile-wrapper, #cf-please-wait, script[src*="challenge-platform"]')) return 'wait'; + if (!document.documentElement || !document.body) return 'wait'; + var contentType = (document.contentType || '').toLowerCase(); + if ((contentType.indexOf('json') !== -1 || contentType.indexOf('text/plain') !== -1) && (document.body.innerText || '').length > 0) return 'ok'; + var html = document.documentElement.outerHTML || ''; + if (html.length < 64) return 'wait'; + return 'ok'; + } catch (e) { + return 'wait'; + } + })() + """ + + private const val PAGE_BODY_JS = """ + (function() { + try { + var contentType = (document.contentType || '').toLowerCase(); + if (contentType.indexOf('json') !== -1 || contentType.indexOf('text/plain') !== -1) { + return document.body ? (document.body.innerText || '') : ''; + } + return document.documentElement ? (document.documentElement.outerHTML || '') : ''; + } catch (e) { + return ''; + } + })() + """ + + private const val PAGE_CONTENT_TYPE_JS = """ + (function() { + try { + return document.contentType || 'text/html'; + } catch (e) { + return 'text/html'; + } + })() + """ + + override suspend fun solve( + url: String, + headers: Map, + forceFresh: Boolean, + timeoutMs: Long, + ): CfSolveResult? { + val host = extractHost(url) + val nsUrl = NSURL.URLWithString(url) ?: run { + log.e { "CF solve: invalid URL: $url" } + return null + } + + if (!forceFresh) { + val existing = getWkCookies(host) + if (existing.containsKey("cf_clearance")) { + return CfSolveResult(cookies = existing, userAgent = getOrCaptureUserAgent()) + } + } + + withContext(Dispatchers.Main) { + clearWkCookiesForHost(host) + } + + var webViewRef: WKWebView? = null + val webViewUserAgent = getOrCaptureUserAgent() + + try { + val delegate = withContext(Dispatchers.Main) { + val d = CloudflareNavigationDelegate() + val config = WKWebViewConfiguration() + val webView = WKWebView( + frame = platform.CoreGraphics.CGRectMake(0.0, 0.0, 375.0, 812.0), + configuration = config, + ) + webView.customUserAgent = webViewUserAgent + webView.navigationDelegate = d + webViewRef = webView + webView.loadRequest(NSURLRequest.requestWithURL(nsUrl)) + d + } + + val result = withTimeoutOrNull(timeoutMs) { + var solved: CfSolveResult? = null + var isFailed = false + while (solved == null && !isFailed) { + delay(100L) + withContext(Dispatchers.Main) { + val error = delegate.error + if (error != null) { + log.e { "CF solve failed due to network/navigation error: ${error.localizedDescription}" } + isFailed = true + return@withContext + } + val cookies = getWkCookies(host) + if (cookies.containsKey("cf_clearance")) { + val finalUrl = webViewRef?.URL?.absoluteString ?: url + solved = CfSolveResult( + cookies = cookies, + userAgent = webViewUserAgent, + redirectUrl = finalUrl.takeIf { it != url }, + ) + } + } + } + solved + } + + if (result == null) { + log.w { "CF solve timed out after ${timeoutMs}ms for $host" } + } + return result + } finally { + withContext(NonCancellable + Dispatchers.Main) { + webViewRef?.navigationDelegate = null + webViewRef?.stopLoading() + webViewRef = null + } + } + } + + override suspend fun fetchRenderedPage( + url: String, + headers: Map, + timeoutMs: Long, + ): WebViewFetchResult? { + val nsUrl = NSURL.URLWithString(url) ?: run { + log.e { "CF fetch fallback: invalid URL: $url" } + return null + } + + var webViewRef: WKWebView? = null + val webViewUserAgent = getOrCaptureUserAgent() + + try { + val delegate = withContext(Dispatchers.Main) { + val d = CloudflareNavigationDelegate() + val config = WKWebViewConfiguration() + val webView = WKWebView( + frame = platform.CoreGraphics.CGRectMake(0.0, 0.0, 375.0, 812.0), + configuration = config, + ) + webView.customUserAgent = webViewUserAgent + webView.navigationDelegate = d + webViewRef = webView + webView.loadRequest(NSURLRequest.requestWithURL(nsUrl)) + d + } + + val result = withTimeoutOrNull(timeoutMs) { + var rendered: WebViewFetchResult? = null + var isFailed = false + while (rendered == null && !isFailed) { + delay(250L) + withContext(Dispatchers.Main) { + val error = delegate.error + if (error != null) { + log.e { "CF fetch fallback failed due to network/navigation error: ${error.localizedDescription}" } + isFailed = true + return@withContext + } + val state = webViewRef?.evaluateJavascriptString(PAGE_STATE_JS).orEmpty() + if (state == "blocked") { + isFailed = true + return@withContext + } + + if (state == "ok") { + val body = webViewRef?.evaluateJavascriptString(PAGE_BODY_JS).orEmpty() + if (body.isNotBlank()) { + val contentType = webViewRef?.evaluateJavascriptString(PAGE_CONTENT_TYPE_JS) + .orEmpty().ifBlank { "text/html" } + val headersWithContentType = delegate.headers.toMutableMap() + if (headersWithContentType.keys.none { it.equals("content-type", ignoreCase = true) }) { + headersWithContentType["content-type"] = contentType + } + rendered = WebViewFetchResult( + status = delegate.status, + statusText = delegate.statusText, + url = webViewRef?.URL?.absoluteString ?: url, + body = body, + headers = headersWithContentType, + ) + } + } + } + } + rendered + } + + if (result == null) { + log.w { "CF WebView fetch fallback timed out after ${timeoutMs}ms for $url" } + } + return result + } finally { + withContext(NonCancellable + Dispatchers.Main) { + webViewRef?.navigationDelegate = null + webViewRef?.stopLoading() + webViewRef = null + } + } + } + + private suspend fun WKWebView.evaluateJavascriptString(script: String): String = + withContext(Dispatchers.Main) { + suspendCoroutine { cont -> + evaluateJavaScript(script) { result, _ -> + cont.resume(result as? String ?: "") + } + } + } + + private suspend fun getOrCaptureUserAgent(): String = withContext(Dispatchers.Main) { + cachedUserAgent?.let { return@withContext it } + val webView = WKWebView( + frame = platform.CoreGraphics.CGRectMake(0.0, 0.0, 375.0, 812.0), + configuration = WKWebViewConfiguration(), + ) + val deferred = CompletableDeferred() + webView.evaluateJavaScript("navigator.userAgent") { result, _ -> + deferred.complete( + result as? String + ?: "Mozilla/5.0 (iPhone; CPU iPhone OS 17_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148", + ) + } + val userAgent = deferred.await() + // Keep the WebView reference alive during suspension by referencing it here + log.d { "Captured User-Agent using WebView. WebView description: ${webView.description}" } + userAgent.also { cachedUserAgent = it } + } + + private suspend fun getWkCookies( + host: String, + ): Map = withContext(Dispatchers.Main) { + val cookieStore = WKWebsiteDataStore.defaultDataStore().httpCookieStore + suspendCoroutine { cont -> + cookieStore.getAllCookies { rawList -> + @Suppress("UNCHECKED_CAST") + val cookies = (rawList as? List) + ?.filter { cookieMatchesHost(it.domain, host) } + ?.associate { it.name to it.value } + ?.filter { (key, value) -> key.isNotBlank() && value.isNotBlank() } + .orEmpty() + cont.resume(cookies) + } + } + } + + private suspend fun clearWkCookiesForHost(host: String) = withContext(Dispatchers.Main) { + suspendCoroutine { cont -> + val store = WKWebsiteDataStore.defaultDataStore().httpCookieStore + store.getAllCookies { rawList -> + @Suppress("UNCHECKED_CAST") + val cookies = (rawList as? List) + ?.filter { cookieMatchesHost(it.domain, host) } + .orEmpty() + + if (cookies.isEmpty()) { + cont.resume(Unit) + return@getAllCookies + } + + var remaining = cookies.size + cookies.forEach { cookie -> + store.deleteCookie(cookie) { + remaining -= 1 + if (remaining == 0) { + cont.resume(Unit) + } + } + } + } + } + } + + private fun cookieMatchesHost(domain: String, host: String): Boolean { + val normalizedDomain = domain.trimStart('.').lowercase() + val normalizedHost = host.lowercase() + return normalizedHost == normalizedDomain || normalizedHost.endsWith(".$normalizedDomain") + } +} + +@OptIn(ExperimentalForeignApi::class) +private class CloudflareNavigationDelegate : NSObject(), WKNavigationDelegateProtocol { + var status: Int = 200 + var statusText: String = "OK" + val headers = mutableMapOf() + var error: NSError? = null + + override fun webView( + webView: WKWebView, + decidePolicyForNavigationResponse: WKNavigationResponse, + decisionHandler: (WKNavigationResponsePolicy) -> Unit + ) { + val httpResponse = decidePolicyForNavigationResponse.response as? NSHTTPURLResponse + if (httpResponse != null) { + status = httpResponse.statusCode.toInt() + statusText = NSHTTPURLResponse.localizedStringForStatusCode(httpResponse.statusCode) + headers.clear() + httpResponse.allHeaderFields.forEach { (key, value) -> + headers[key.toString()] = value.toString() + } + } + decisionHandler(WKNavigationResponsePolicy.WKNavigationResponsePolicyAllow) + } + + @ObjCSignatureOverride + override fun webView( + webView: WKWebView, + didFailNavigation: WKNavigation?, + withError: NSError + ) { + error = withError + } + + @ObjCSignatureOverride + override fun webView( + webView: WKWebView, + didFailProvisionalNavigation: WKNavigation?, + withError: NSError + ) { + error = withError + } + + override fun webView( + webView: WKWebView, + didReceiveAuthenticationChallenge: platform.Foundation.NSURLAuthenticationChallenge, + completionHandler: (platform.Foundation.NSURLSessionAuthChallengeDisposition, platform.Foundation.NSURLCredential?) -> Unit + ) { + val trust = didReceiveAuthenticationChallenge.protectionSpace?.serverTrust + if (trust != null) { + completionHandler( + platform.Foundation.NSURLSessionAuthChallengeUseCredential, + NSURLCredential.credentialForTrust(trust) + ) + } else { + completionHandler(platform.Foundation.NSURLSessionAuthChallengePerformDefaultHandling, null) + } + } +}