From caebb0a5f93e9c8c7f9cd6cd38910ea68ea63a53 Mon Sep 17 00:00:00 2001 From: 666mxvbee Date: Sun, 31 May 2026 03:49:27 +0300 Subject: [PATCH 1/2] hotfix: increased a time to check the updates due to 403github failure --- src/LinkTracker.Scrapper/appsettings.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/LinkTracker.Scrapper/appsettings.json b/src/LinkTracker.Scrapper/appsettings.json index 08b02d0..ee8df37 100644 --- a/src/LinkTracker.Scrapper/appsettings.json +++ b/src/LinkTracker.Scrapper/appsettings.json @@ -11,9 +11,9 @@ "RunMigrations": true }, "Scrapper": { - "CheckIntervalSeconds": 30, + "CheckIntervalSeconds": 300, "BatchSize": 100, - "Parallelism": 4, + "Parallelism": 2, "GitHubBaseUrl": "https://api.github.com/", "StackOverflowBaseUrl": "https://api.stackexchange.com/2.3/" }, From 77646d0a5c5de99a986bc6ba82bbc55fc8c56a72 Mon Sep 17 00:00:00 2001 From: 666mxvbee Date: Sat, 6 Jun 2026 20:18:53 +0300 Subject: [PATCH 2/2] feat: add github token --- .env.example | 3 ++- docker-compose.yml | 1 + .../Configuration/ScrapperOptions.cs | 2 ++ src/LinkTracker.Scrapper/Program.cs | 9 +++++++++ 4 files changed, 14 insertions(+), 1 deletion(-) diff --git a/.env.example b/.env.example index 8eec027..7fe65e8 100644 --- a/.env.example +++ b/.env.example @@ -2,4 +2,5 @@ POSTGRES_DB=linktracker POSTGRES_USER=linktracker -POSTGRES_PASSWORD=linktracker \ No newline at end of file +POSTGRES_PASSWORD=linktracker +GITHUB_TOKEN=your-github-token \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 7e2fd40..6393ef3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -25,6 +25,7 @@ services: - Database__AccessType=SQL - Database__ConnectionString=Host=postgres;Port=5432;Database=${POSTGRES_DB};Username=${POSTGRES_USER};Password=${POSTGRES_PASSWORD} - Database__RunMigrations=true + - Scrapper__GitHubToken=${GITHUB_TOKEN} depends_on: postgres: condition: service_healthy diff --git a/src/LinkTracker.Scrapper/Configuration/ScrapperOptions.cs b/src/LinkTracker.Scrapper/Configuration/ScrapperOptions.cs index 6586c44..6a49310 100644 --- a/src/LinkTracker.Scrapper/Configuration/ScrapperOptions.cs +++ b/src/LinkTracker.Scrapper/Configuration/ScrapperOptions.cs @@ -10,4 +10,6 @@ public sealed class ScrapperOptions public string GitHubBaseUrl { get; init; } = "https://api.github.com/"; public string StackOverflowBaseUrl { get; init; } = "https://api.stackexchange.com/2.3/"; + + public string? GitHubToken { get; init; } } \ No newline at end of file diff --git a/src/LinkTracker.Scrapper/Program.cs b/src/LinkTracker.Scrapper/Program.cs index dc99152..f3c8815 100644 --- a/src/LinkTracker.Scrapper/Program.cs +++ b/src/LinkTracker.Scrapper/Program.cs @@ -8,6 +8,7 @@ using LinkTracker.Scrapper.Services.Notifications; using LinkTracker.Scrapper.Services.Updates; using Microsoft.EntityFrameworkCore; +using System.Net.Http.Headers; using Npgsql; using Quartz; @@ -51,6 +52,14 @@ { client.BaseAddress = new Uri(scrapperOptions.GitHubBaseUrl); client.DefaultRequestHeaders.UserAgent.ParseAdd("LinkTrackerBot/1.0"); + client.DefaultRequestHeaders.Accept.ParseAdd("application/vnd.github+json"); + client.DefaultRequestHeaders.Add("X-GitHub-Api-Version", "2022-11-28"); + + if (!string.IsNullOrWhiteSpace(scrapperOptions.GitHubToken)) + { + client.DefaultRequestHeaders.Authorization = + new AuthenticationHeaderValue("Bearer", scrapperOptions.GitHubToken); + } }); builder.Services.AddHttpClient(client =>