From 60ae549d131113b64a878720332bc198f3fde4a5 Mon Sep 17 00:00:00 2001 From: Jacobo de Vera Date: Sat, 20 Jul 2024 01:11:13 +0100 Subject: [PATCH 1/5] Add new DRY_RUN parameter that prevents writing to gitea This is so that the script can be tested more easily without introducing changes in the destination Gitea instance. --- README.md | 6 ++++-- src/index.js | 9 +++++++-- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 0acc868..74af01b 100644 --- a/README.md +++ b/README.md @@ -53,9 +53,10 @@ services: - GITHUB_USERNAME=github-user - GITEA_URL=https://your-gitea.url - GITEA_TOKEN=please-exchange-with-token - #- GITHUB_TOKEN=please-exchange-with-token # Optional, set to mirror private repos - #- MIRROR_PRIVATE_REPOSITORIES=true # Optional, set to mirror private repos + # - GITHUB_TOKEN=please-exchange-with-token # Optional, set to mirror private repos + # - MIRROR_PRIVATE_REPOSITORIES=true # Optional, set to mirror private repos # - DELAY=3600 # Optional, set to change the delay between checks (in seconds) + # - DRY_RUN=true # Optional, set to only log what would be done container_name: mirror-to-gitea ``` ## Building from Source @@ -99,6 +100,7 @@ In your Docker Compose file, replace `jaedle/mirror-to-gitea:latest` with `build - `GITHUB_TOKEN`: [GitHub personal access token](https://docs.github.com/en/free-pro-team@latest/github/authenticating-to-github/creating-a-personal-access-token). **Attention: if this is set, the token will be transmitted to your specified Gitea instance!** - `MIRROR_PRIVATE_REPOSITORIES`: If set to `true`, your private GitHub repositories will also be mirrored to gitea. The `GITHUB_TOKEN` parameter must be set for this to work. - `DELAY`: How often to check for new repositories in seconds. Default is 3600 (1 hour). +- `DRY_RUN`: If set to `true` or `1`, the script will only log what would be done, but not actually create any mirror. ## Things to do diff --git a/src/index.js b/src/index.js index f7feab6..ee16f46 100644 --- a/src/index.js +++ b/src/index.js @@ -80,13 +80,17 @@ function mirrorOnGitea(repository, gitea, giteaUser, githubToken) { } -async function mirror(repository, gitea, giteaUser, githubToken) { +async function mirror(repository, gitea, giteaUser, githubToken, dryRun) { if (await isAlreadyMirroredOnGitea(repository.name, gitea, giteaUser)) { console.log('Repository is already mirrored; doing nothing: ', repository.name); return; } + if (dryRun) { + console.log('DRY RUN: Would mirror repository to gitea: ', repository); + return; + } console.log('Mirroring repository to gitea: ', repository.name); await mirrorOnGitea(repository, gitea, giteaUser, githubToken); } @@ -117,6 +121,7 @@ async function main() { return; } + const dryRun = ['1', 'true'].includes(process.env.DRY_RUN); const githubRepositories = await getGithubRepositories(githubUsername, githubToken, mirrorPrivateRepositories); console.log(`Found ${githubRepositories.length} repositories on github`); @@ -130,7 +135,7 @@ async function main() { const queue = new PQueue({ concurrency: 4 }); await queue.addAll(githubRepositories.map(repository => { return async () => { - await mirror(repository, gitea, giteaUser, githubToken); + await mirror(repository, gitea, giteaUser, githubToken, dryRun); }; })); } From 81e11cde9b776c2f359f1bb3bcf336daed6cd704 Mon Sep 17 00:00:00 2001 From: Jacobo de Vera Date: Sat, 20 Jul 2024 01:11:41 +0100 Subject: [PATCH 2/5] Resolve env flags to boolean early So inner functions can assume booleans and forget about the fact that they were once strings. Also allow both true and 1 to set it. --- src/index.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/index.js b/src/index.js index ee16f46..360578a 100644 --- a/src/index.js +++ b/src/index.js @@ -115,8 +115,8 @@ async function main() { return; } - const mirrorPrivateRepositories = process.env.MIRROR_PRIVATE_REPOSITORIES; - if(mirrorPrivateRepositories === 'true' && !githubToken){ + const mirrorPrivateRepositories = ['1', 'true'].includes(process.env.MIRROR_PRIVATE_REPOSITORIES) + if(mirrorPrivateRepositories && !githubToken){ console.error('MIRROR_PRIVATE_REPOSITORIES was set to true but no GITHUB_TOKEN was specified, please specify! Exiting..') return; } From 3e8eab36551945f224b8974bf2dc79dd7244b468 Mon Sep 17 00:00:00 2001 From: Jacobo de Vera Date: Sat, 20 Jul 2024 01:11:41 +0100 Subject: [PATCH 3/5] Allow filtering out forks when mirroring --- README.md | 4 +++- src/index.js | 29 ++++++++++++++++++----------- 2 files changed, 21 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 74af01b..ad57d54 100644 --- a/README.md +++ b/README.md @@ -56,6 +56,7 @@ services: # - GITHUB_TOKEN=please-exchange-with-token # Optional, set to mirror private repos # - MIRROR_PRIVATE_REPOSITORIES=true # Optional, set to mirror private repos # - DELAY=3600 # Optional, set to change the delay between checks (in seconds) + # - SKIP_FORKS=true # Optional, set to skip forks # - DRY_RUN=true # Optional, set to only log what would be done container_name: mirror-to-gitea ``` @@ -98,7 +99,8 @@ In your Docker Compose file, replace `jaedle/mirror-to-gitea:latest` with `build ### Optional - `GITHUB_TOKEN`: [GitHub personal access token](https://docs.github.com/en/free-pro-team@latest/github/authenticating-to-github/creating-a-personal-access-token). **Attention: if this is set, the token will be transmitted to your specified Gitea instance!** -- `MIRROR_PRIVATE_REPOSITORIES`: If set to `true`, your private GitHub repositories will also be mirrored to gitea. The `GITHUB_TOKEN` parameter must be set for this to work. +- `MIRROR_PRIVATE_REPOSITORIES`: If set to `true` or `1`, your private GitHub repositories will also be mirrored to gitea. The `GITHUB_TOKEN` parameter must be set for this to work. +- `SKIP_FORKS`: If set to `true` or `1`, forks will NOT be mirrored. - `DELAY`: How often to check for new repositories in seconds. Default is 3600 (1 hour). - `DRY_RUN`: If set to `true` or `1`, the script will only log what would be done, but not actually create any mirror. diff --git a/src/index.js b/src/index.js index 360578a..3d84e64 100644 --- a/src/index.js +++ b/src/index.js @@ -3,30 +3,36 @@ const request = require('superagent'); const {default: PQueue} = require('p-queue'); -async function getGithubRepositories(username, token, mirrorPrivateRepositories) { +async function getGithubRepositories(username, token, mirrorPrivateRepositories, mirrorForks) { const octokit = new Octokit({ auth: token || null, }); - const publicRepositoriesWithForks = await octokit.paginate('GET /users/:username/repos', { username: username }) + const publicRepositories = await octokit.paginate('GET /users/:username/repos', { username: username }) .then(repositories => toRepositoryList(repositories)); - let allRepositoriesWithoutForks; - if(mirrorPrivateRepositories === 'true'){ - allRepositoriesWithoutForks = await octokit.paginate('GET /user/repos?visibility=public&affiliation=owner&visibility=private') + let allOwnedRepositories; + if(mirrorPrivateRepositories){ + allOwnedRepositories = await octokit.paginate('GET /user/repos?visibility=public&affiliation=owner&visibility=private') .then(repositories => toRepositoryList(repositories)); } - if(mirrorPrivateRepositories === 'true'){ - return filterDuplicates(allRepositoriesWithoutForks.concat(publicRepositoriesWithForks)); - }else{ - return publicRepositoriesWithForks; + let repositories = publicRepositories; + + if(mirrorPrivateRepositories) { + repositories = filterDuplicates(allOwnedRepositories.concat(publicRepositories)); + } + + if(!mirrorForks){ + repositories = repositories.filter(repository => !repository.fork); } + + return repositories; } function toRepositoryList(repositories) { return repositories.map(repository => { - return { name: repository.name, url: repository.clone_url, private: repository.private }; + return { name: repository.name, url: repository.clone_url, private: repository.private, fork: repository.fork}; }); } @@ -101,6 +107,7 @@ async function main() { console.error('No GITHUB_USERNAME specified, please specify! Exiting..'); return; } + const mirrorForks = ! ['1', 'true'].includes(process.env.SKIP_FORKS); const githubToken = process.env.GITHUB_TOKEN; const giteaUrl = process.env.GITEA_URL; @@ -123,7 +130,7 @@ async function main() { const dryRun = ['1', 'true'].includes(process.env.DRY_RUN); - const githubRepositories = await getGithubRepositories(githubUsername, githubToken, mirrorPrivateRepositories); + const githubRepositories = await getGithubRepositories(githubUsername, githubToken, mirrorPrivateRepositories, mirrorForks); console.log(`Found ${githubRepositories.length} repositories on github`); const gitea = { From 11f95daaf895f15ebd7ee1e838798bb8a8ac4c33 Mon Sep 17 00:00:00 2001 From: Jacobo de Vera Date: Sat, 20 Jul 2024 01:11:41 +0100 Subject: [PATCH 4/5] Prefer unless-stopped for restart policy Otherwise the explicit user action of stopping it will be undone on next start. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index ad57d54..029a246 100644 --- a/README.md +++ b/README.md @@ -48,7 +48,7 @@ version: "3.3" services: mirror-to-gitea: image: jaedle/mirror-to-gitea:latest - restart: always + restart: unless-stopped environment: - GITHUB_USERNAME=github-user - GITEA_URL=https://your-gitea.url From 921713518d65322ff65afca67c775e6627af0bdc Mon Sep 17 00:00:00 2001 From: Jacobo de Vera Date: Sat, 20 Jul 2024 01:11:41 +0100 Subject: [PATCH 5/5] Log the start of each run with its parameters But do not show secrets. --- src/index.js | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/index.js b/src/index.js index 3d84e64..e3a1df7 100644 --- a/src/index.js +++ b/src/index.js @@ -130,6 +130,16 @@ async function main() { const dryRun = ['1', 'true'].includes(process.env.DRY_RUN); + console.log("Starting with the following configuration:") + console.log(` - GITHUB_USERNAME: ${githubUsername}`); + console.log(` - GITHUB_TOKEN: ${githubToken ? '****' : ''}`); + console.log(` - GITEA_URL: ${giteaUrl}`); + console.log(` - GITEA_TOKEN: ${giteaToken ? '****' : ''}`); + console.log(` - MIRROR_PRIVATE_REPOSITORIES: ${mirrorPrivateRepositories}`); + console.log(` - SKIP_FORKS: ${!mirrorForks}`); + console.log(` - DRY_RUN: ${dryRun}`); + + const githubRepositories = await getGithubRepositories(githubUsername, githubToken, mirrorPrivateRepositories, mirrorForks); console.log(`Found ${githubRepositories.length} repositories on github`);