-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathfetch_games.ts
More file actions
67 lines (53 loc) · 2.24 KB
/
fetch_games.ts
File metadata and controls
67 lines (53 loc) · 2.24 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
import { write } from "bun";
import { mkdir } from "node:fs/promises";
const URLs = [
"https://www.4j.com/ajax_loadmoregames.php?from=0&num=120000&type=hot",
"https://www.4j.com/ajax_loadmoregames.php?from=0&num=1000000002&type=best"
];
const OUTPUT_DIR = "games";
async function fetchFromUrl(url: string) {
console.log(`Fetching data from ${url}...`);
try {
const response = await fetch(url);
if (!response.ok) {
throw new Error(`HTTP error! status: ${response.status}`);
}
const htmlContent = await response.text();
console.log(`Successfully fetched ${htmlContent.length} bytes of data.`);
// Regex to match game data
const pattern = /<div class="thumb" id='game-(\d+)'><a href="([^"]+)"><img src='([^']+)' alt="[^"]*" \/><span class='GameName'>(.*?)<\/span><span class='GameRating'>([\d.]+)<\/span><\/a><\/div>/g;
let match;
let count = 0;
while ((match = pattern.exec(htmlContent)) !== null) {
const [_, gameId, gameUrlSuffix, imageUrl, gameName, rating] = match;
const slug = gameUrlSuffix.startsWith('/') ? gameUrlSuffix.slice(1) : gameUrlSuffix;
const gameData = {
id: gameId,
name: gameName,
slug: slug,
url: `https://www.4j.com${gameUrlSuffix}`,
embedUrl: `https://www.4j.com/embed${gameUrlSuffix}`,
image: imageUrl,
rating: rating,
};
const outputPath = `${OUTPUT_DIR}/${gameId}.json`;
await write(outputPath, JSON.stringify(gameData, null, 2));
count++;
}
console.log(`Successfully processed ${count} games from URL.`);
return count;
} catch (error) {
console.error(`Error fetching from ${url}:`, error);
return 0;
}
}
async function fetchAndSaveGames() {
console.log(`Creating directory: ${OUTPUT_DIR}`);
await mkdir(OUTPUT_DIR, { recursive: true });
let totalGames = 0;
for (const url of URLs) {
totalGames += await fetchFromUrl(url);
}
console.log(`Finished. Processed potential total of ${totalGames} game entries (duplicates handled by file overwrite).`);
}
fetchAndSaveGames();