Skip to content

Commit 8bd7261

Browse files
rainxchzedrainxchzed
committed
feat: implement two-phase loading for topic repositories with caching
- Introduce a pre-fetch phase in `HomeViewModel` to load cached topic repositories instantly before supplementing with live GitHub search results. - Implement `getCachedTopicRepos` in `CachedRepositoriesDataSource` to fetch and merge platform-specific repository data from a remote JSON source. - Add a `topicMemoryCache` with TTL to `CachedRepositoriesDataSourceImpl` to reduce redundant network requests. - Update `HomeRepository` to support fetching topic-specific cached repositories via a new `getTopicRepositories` flow. - Enhance data merging logic to handle repository duplicates across different platform files (Android, Windows, MacOS, Linux) and preserve the latest release dates and platform availability.
1 parent 5298a7a commit 8bd7261

5 files changed

Lines changed: 177 additions & 0 deletions

File tree

‎feature/home/data/src/commonMain/kotlin/zed/rainxch/home/data/data_source/CachedRepositoriesDataSource.kt‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,11 +2,14 @@ package zed.rainxch.home.data.data_source
22

33
import zed.rainxch.core.domain.model.DiscoveryPlatform
44
import zed.rainxch.home.data.dto.CachedRepoResponse
5+
import zed.rainxch.home.domain.model.TopicCategory
56

67
interface CachedRepositoriesDataSource {
78
suspend fun getCachedTrendingRepos(platform: DiscoveryPlatform): CachedRepoResponse?
89

910
suspend fun getCachedHotReleaseRepos(platform: DiscoveryPlatform): CachedRepoResponse?
1011

1112
suspend fun getCachedMostPopularRepos(platform: DiscoveryPlatform): CachedRepoResponse?
13+
14+
suspend fun getCachedTopicRepos(topic: TopicCategory, platform: DiscoveryPlatform): CachedRepoResponse?
1215
}

‎feature/home/data/src/commonMain/kotlin/zed/rainxch/home/data/data_source/impl/CachedRepositoriesDataSourceImpl.kt‎

Lines changed: 124 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ import zed.rainxch.home.data.data_source.CachedRepositoriesDataSource
2121
import zed.rainxch.home.data.dto.CachedGithubRepoSummary
2222
import zed.rainxch.home.data.dto.CachedRepoResponse
2323
import zed.rainxch.home.domain.model.HomeCategory
24+
import zed.rainxch.home.domain.model.TopicCategory
2425
import kotlin.coroutines.cancellation.CancellationException
2526
import kotlin.time.Clock
2627
import kotlin.time.Duration.Companion.hours
@@ -49,6 +50,7 @@ class CachedRepositoriesDataSourceImpl(
4950

5051
private val cacheMutex = Mutex()
5152
private val memoryCache = mutableMapOf<CacheKey, CacheEntry>()
53+
private val topicMemoryCache = mutableMapOf<TopicCacheKey, CacheEntry>()
5254

5355
private data class CacheEntry(
5456
val data: CachedRepoResponse,
@@ -64,6 +66,123 @@ class CachedRepositoriesDataSourceImpl(
6466
override suspend fun getCachedMostPopularRepos(platform: DiscoveryPlatform): CachedRepoResponse? =
6567
fetchCachedReposForCategory(platform, HomeCategory.MOST_POPULAR)
6668

69+
override suspend fun getCachedTopicRepos(
70+
topic: TopicCategory,
71+
platform: DiscoveryPlatform,
72+
): CachedRepoResponse? {
73+
val topicFolder = when (topic) {
74+
TopicCategory.PRIVACY -> "privacy"
75+
TopicCategory.MEDIA -> "media"
76+
TopicCategory.PRODUCTIVITY -> "productivity"
77+
TopicCategory.NETWORKING -> "networking"
78+
TopicCategory.DEV_TOOLS -> "dev-tools"
79+
}
80+
81+
val topicCacheKey = TopicCacheKey(topic, platform)
82+
val cached = cacheMutex.withLock { topicMemoryCache[topicCacheKey] }
83+
if (cached != null) {
84+
val age = Clock.System.now() - cached.fetchedAt
85+
if (age < CACHE_TTL) {
86+
logger.debug("Topic memory cache hit for $topicCacheKey (age: ${age.inWholeSeconds}s)")
87+
return cached.data
88+
}
89+
}
90+
91+
return withContext(Dispatchers.IO) {
92+
val paths = listOf(
93+
"cached-data/topics/$topicFolder/android.json",
94+
"cached-data/topics/$topicFolder/windows.json",
95+
"cached-data/topics/$topicFolder/macos.json",
96+
"cached-data/topics/$topicFolder/linux.json",
97+
)
98+
99+
val responses = coroutineScope {
100+
paths.map { path ->
101+
async {
102+
val url = "https://raw.githubusercontent.com/OpenHub-Store/api/main/$path"
103+
val filePlatform = when {
104+
path.contains("/android") -> DiscoveryPlatform.Android
105+
path.contains("/windows") -> DiscoveryPlatform.Windows
106+
path.contains("/macos") -> DiscoveryPlatform.Macos
107+
path.contains("/linux") -> DiscoveryPlatform.Linux
108+
else -> error("Unknown platform in path: $path")
109+
}
110+
try {
111+
logger.debug("Fetching topic cache: $url")
112+
val response: HttpResponse = httpClient.get(url)
113+
if (response.status.isSuccess()) {
114+
json.decodeFromString<CachedRepoResponse>(response.bodyAsText())
115+
.let { repoResponse ->
116+
repoResponse.copy(
117+
repositories = repoResponse.repositories.map {
118+
it.copy(availablePlatforms = listOf(filePlatform))
119+
},
120+
)
121+
}
122+
} else {
123+
logger.error("HTTP ${response.status.value} from $url")
124+
null
125+
}
126+
} catch (e: SerializationException) {
127+
logger.error("Parse error from $url: ${e.message}")
128+
null
129+
} catch (e: CancellationException) {
130+
throw e
131+
} catch (e: Exception) {
132+
logger.error("Error with $url: ${e.message}")
133+
null
134+
}
135+
}
136+
}.awaitAll().filterNotNull()
137+
}
138+
139+
if (responses.isEmpty()) {
140+
logger.error("All topic mirrors failed for $topicCacheKey")
141+
return@withContext null
142+
}
143+
144+
val allMergedRepos = responses
145+
.asSequence()
146+
.flatMap { it.repositories.asSequence() }
147+
.groupBy { it.id }
148+
.values
149+
.map { duplicates ->
150+
duplicates.reduce { acc, repo ->
151+
acc.copy(
152+
availablePlatforms = (acc.availablePlatforms + repo.availablePlatforms).distinct(),
153+
latestReleaseDate = listOfNotNull(
154+
acc.latestReleaseDate,
155+
repo.latestReleaseDate,
156+
).maxOrNull(),
157+
)
158+
}
159+
}
160+
.sortedByDescending { it.stargazersCount }
161+
162+
val filteredRepos = when (platform) {
163+
DiscoveryPlatform.All -> allMergedRepos
164+
else -> allMergedRepos.filter { platform in it.availablePlatforms }
165+
}.toList()
166+
167+
val merged = CachedRepoResponse(
168+
category = "topic",
169+
platform = platform.name.lowercase(),
170+
lastUpdated = responses.maxOf { it.lastUpdated },
171+
totalCount = filteredRepos.size,
172+
repositories = filteredRepos,
173+
)
174+
175+
if (responses.size == paths.size) {
176+
cacheMutex.withLock {
177+
topicMemoryCache[topicCacheKey] =
178+
CacheEntry(data = merged, fetchedAt = Clock.System.now())
179+
}
180+
}
181+
182+
merged
183+
}
184+
}
185+
67186
private suspend fun fetchCachedReposForCategory(
68187
platform: DiscoveryPlatform,
69188
category: HomeCategory,
@@ -230,4 +349,9 @@ class CachedRepositoriesDataSourceImpl(
230349
val platform: DiscoveryPlatform,
231350
val category: HomeCategory,
232351
)
352+
353+
private data class TopicCacheKey(
354+
val topic: TopicCategory,
355+
val platform: DiscoveryPlatform,
356+
)
233357
}

‎feature/home/data/src/commonMain/kotlin/zed/rainxch/home/data/repository/HomeRepositoryImpl.kt‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -269,6 +269,27 @@ class HomeRepositoryImpl(
269269
)
270270
}.flowOn(Dispatchers.IO)
271271

272+
override fun getTopicRepositories(
273+
topic: zed.rainxch.home.domain.model.TopicCategory,
274+
platform: DiscoveryPlatform,
275+
): Flow<PaginatedDiscoveryRepositories> =
276+
flow {
277+
val cachedData = cachedDataSource.getCachedTopicRepos(topic, platform)
278+
279+
if (cachedData != null && cachedData.repositories.isNotEmpty()) {
280+
logger.debug("Using cached topic data for ${topic.name}: ${cachedData.repositories.size} repos")
281+
282+
val repos = cachedData.repositories.map { it.toGithubRepoSummary() }
283+
emit(
284+
PaginatedDiscoveryRepositories(
285+
repos = repos,
286+
hasMore = false,
287+
nextPageIndex = 2,
288+
),
289+
)
290+
}
291+
}.flowOn(Dispatchers.IO)
292+
272293
override fun searchByTopic(
273294
searchKeywords: String,
274295
platform: DiscoveryPlatform,

‎feature/home/domain/src/commonMain/kotlin/zed/rainxch/home/domain/repository/HomeRepository.kt‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ package zed.rainxch.home.domain.repository
33
import kotlinx.coroutines.flow.Flow
44
import zed.rainxch.core.domain.model.DiscoveryPlatform
55
import zed.rainxch.core.domain.model.PaginatedDiscoveryRepositories
6+
import zed.rainxch.home.domain.model.TopicCategory
67

78
interface HomeRepository {
89
fun getTrendingRepositories(
@@ -25,4 +26,9 @@ interface HomeRepository {
2526
platform: DiscoveryPlatform,
2627
page: Int,
2728
): Flow<PaginatedDiscoveryRepositories>
29+
30+
fun getTopicRepositories(
31+
topic: TopicCategory,
32+
platform: DiscoveryPlatform,
33+
): Flow<PaginatedDiscoveryRepositories>
2834
}

‎feature/home/presentation/src/commonMain/kotlin/zed/rainxch/home/presentation/HomeViewModel.kt‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -259,6 +259,29 @@ class HomeViewModel(
259259
_state.update { it.copy(isLoadingTopicSupplement = true) }
260260

261261
try {
262+
// Phase 1: Load pre-fetched cached topic repos (instant, no API cost)
263+
homeRepository
264+
.getTopicRepositories(
265+
topic = topic,
266+
platform = platform,
267+
).collect { paginatedRepos ->
268+
if (paginatedRepos.repos.isNotEmpty()) {
269+
val cachedReposWithStatus = mapReposToUi(paginatedRepos.repos)
270+
271+
_state.update { currentState ->
272+
val merged = (currentState.repos + cachedReposWithStatus)
273+
.distinctBy { it.repository.fullName }
274+
275+
currentState.copy(
276+
repos = merged.toImmutableList(),
277+
)
278+
}
279+
280+
logger.debug("Loaded ${paginatedRepos.repos.size} cached topic repos for ${topic.name}")
281+
}
282+
}
283+
284+
// Phase 2: Supplement with live GitHub search (fills gaps)
262285
homeRepository
263286
.searchByTopic(
264287
searchKeywords = topic.searchKeywords,

0 commit comments

Comments
 (0)