From 26ed54e854ca7b26d04108752233e96212bb362a Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Mon, 23 Jun 2025 17:58:12 +0300 Subject: Add support for multiple repository configuration and sync MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add optional 'repositories' array to configuration file - Add --list-repos flag to list configured repositories - Add --sync-all flag to sync all configured repositories at once - Show progress when syncing multiple repositories - Gracefully handle missing remote repositories with warnings - Improve error handling to continue syncing other repos on failure - Add comprehensive integration tests for all functionality - Add test for multiple repository sync feature Example usage: gitsyncer --sync-all # Sync all configured repos gitsyncer --list-repos # List configured repos gitsyncer --sync repo-name # Sync specific repo (still works) 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude --- cmd/gitsyncer/main.go | 58 ++++++++++- gitsyncer | Bin 3383664 -> 3399298 bytes gitsyncer.example.json | 5 + internal/config/config.go | 1 + internal/sync/sync.go | 71 +++++++++++-- test/run_integration_tests.sh | 229 ++++++++++++++++++++++++++++++++++++++++++ test/test_multi_repo.sh | 102 +++++++++++++++++++ 7 files changed, 458 insertions(+), 8 deletions(-) create mode 100755 test/run_integration_tests.sh create mode 100755 test/test_multi_repo.sh diff --git a/cmd/gitsyncer/main.go b/cmd/gitsyncer/main.go index dfee39c..63ce0b0 100644 --- a/cmd/gitsyncer/main.go +++ b/cmd/gitsyncer/main.go @@ -17,7 +17,9 @@ func main() { versionFlag bool configPath string listOrgs bool + listRepos bool syncRepo string + syncAll bool workDir string ) @@ -27,7 +29,9 @@ func main() { flag.StringVar(&configPath, "config", "", "path to configuration file") flag.StringVar(&configPath, "c", "", "path to configuration file (short)") flag.BoolVar(&listOrgs, "list-orgs", false, "list configured organizations") + flag.BoolVar(&listRepos, "list-repos", false, "list configured repositories") flag.StringVar(&syncRepo, "sync", "", "repository name to sync") + flag.BoolVar(&syncAll, "sync-all", false, "sync all configured repositories") flag.StringVar(&workDir, "work-dir", ".gitsyncer-work", "working directory for cloning repositories") flag.Parse() @@ -70,6 +74,10 @@ func main() { "organizations": [ {"host": "git@github.com", "name": "myorg"}, {"host": "git@codeberg.org", "name": "myorg"} + ], + "repositories": [ + "repo1", + "repo2" ] }`) os.Exit(1) @@ -93,6 +101,19 @@ func main() { os.Exit(0) } + // Handle list repositories flag + if listRepos { + fmt.Println("\nConfigured repositories:") + if len(cfg.Repositories) == 0 { + fmt.Println(" (none configured)") + } else { + for _, repo := range cfg.Repositories { + fmt.Printf(" - %s\n", repo) + } + } + os.Exit(0) + } + // Handle sync operation if syncRepo != "" { syncer := sync.New(cfg, workDir) @@ -102,12 +123,45 @@ func main() { os.Exit(0) } + // Handle sync all operation + if syncAll { + if len(cfg.Repositories) == 0 { + fmt.Println("No repositories configured. Add repositories to the config file.") + os.Exit(1) + } + + syncer := sync.New(cfg, workDir) + failedRepos := []string{} + + for i, repo := range cfg.Repositories { + fmt.Printf("\n[%d/%d] Syncing %s...\n", i+1, len(cfg.Repositories), repo) + if err := syncer.SyncRepository(repo); err != nil { + fmt.Printf("Failed to sync %s: %v\n", repo, err) + failedRepos = append(failedRepos, repo) + } + } + + if len(failedRepos) > 0 { + fmt.Printf("\nFailed to sync %d repository(ies):\n", len(failedRepos)) + for _, repo := range failedRepos { + fmt.Printf(" - %s\n", repo) + } + os.Exit(1) + } + + fmt.Printf("\nSuccessfully synced all %d repositories!\n", len(cfg.Repositories)) + os.Exit(0) + } + // Default: show usage fmt.Println("\ngitsyncer - Git repository synchronization tool") - fmt.Printf("Configured with %d organization(s)\n", len(cfg.Organizations)) + fmt.Printf("Configured with %d organization(s) and %d repository(ies)\n", + len(cfg.Organizations), len(cfg.Repositories)) fmt.Println("\nUsage:") - fmt.Println(" gitsyncer --sync Sync a repository across all organizations") + fmt.Println(" gitsyncer --sync Sync a specific repository") + fmt.Println(" gitsyncer --sync-all Sync all configured repositories") fmt.Println(" gitsyncer --list-orgs List configured organizations") + fmt.Println(" gitsyncer --list-repos List configured repositories") fmt.Println(" gitsyncer --version Show version information") fmt.Println("\nOptions:") fmt.Println(" --config Path to configuration file") diff --git a/gitsyncer b/gitsyncer index d8f95c8..ae851db 100755 Binary files a/gitsyncer and b/gitsyncer differ diff --git a/gitsyncer.example.json b/gitsyncer.example.json index 3170286..9116f35 100644 --- a/gitsyncer.example.json +++ b/gitsyncer.example.json @@ -8,5 +8,10 @@ "host": "git@github.com", "name": "snonux" } + ], + "repositories": [ + "gitsyncer", + "another-repo", + "yet-another-repo" ] } \ No newline at end of file diff --git a/internal/config/config.go b/internal/config/config.go index 0fe1ac8..754e226 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -17,6 +17,7 @@ type Organization struct { // Config holds the application configuration type Config struct { Organizations []Organization `json:"organizations"` + Repositories []string `json:"repositories,omitempty"` } // Load reads and parses the configuration file diff --git a/internal/sync/sync.go b/internal/sync/sync.go index 4fe1b13..e35dad7 100644 --- a/internal/sync/sync.go +++ b/internal/sync/sync.go @@ -69,6 +69,24 @@ func (s *Syncer) SyncRepository(repoName string) error { return fmt.Errorf("failed to add remote %s: %w", s.getRemoteName(org), err) } } + } else { + // Repository exists, ensure all remotes are configured + fmt.Printf("Using existing repository at %s\n", repoPath) + + // Check and add any missing remotes + for i := range s.config.Organizations { + org := &s.config.Organizations[i] + remoteName := s.getRemoteName(org) + + // Check if remote exists + cmd := exec.Command("git", "-C", repoPath, "remote", "get-url", remoteName) + if err := cmd.Run(); err != nil { + // Remote doesn't exist, add it + if err := s.addRemote(repoPath, org); err != nil { + return fmt.Errorf("failed to add remote %s: %w", remoteName, err) + } + } + } } // Change to repository directory @@ -155,10 +173,43 @@ func (s *Syncer) addRemote(repoPath string, org *config.Organization) error { // fetchAll fetches from all remotes func (s *Syncer) fetchAll() error { - cmd := exec.Command("git", "fetch", "--all", "--prune") - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - return cmd.Run() + // First, check which remotes actually exist + cmd := exec.Command("git", "remote", "-v") + output, err := cmd.Output() + if err != nil { + return fmt.Errorf("failed to list remotes: %w", err) + } + + // Try to fetch from each remote individually to handle missing repos + remotes := make(map[string]bool) + lines := strings.Split(string(output), "\n") + for _, line := range lines { + if line == "" { + continue + } + parts := strings.Fields(line) + if len(parts) >= 1 { + remotes[parts[0]] = true + } + } + + // Fetch from each remote + for remote := range remotes { + fmt.Printf("Fetching %s\n", remote) + cmd := exec.Command("git", "fetch", remote, "--prune") + output, err := cmd.CombinedOutput() + if err != nil { + // Check if it's because the repository doesn't exist + if strings.Contains(string(output), "does not appear to be a git repository") || + strings.Contains(string(output), "Could not read from remote repository") { + fmt.Printf(" Warning: Remote repository %s does not exist yet\n", remote) + continue + } + return fmt.Errorf("failed to fetch from %s: %w\n%s", remote, err, string(output)) + } + } + + return nil } // getAllBranches gets all unique branches from all remotes @@ -242,8 +293,16 @@ func (s *Syncer) syncBranch(branch string, remotes map[string]*config.Organizati output, err := cmd.CombinedOutput() if err != nil { + outputStr := string(output) + // Check if it's because the repository doesn't exist + if strings.Contains(outputStr, "does not appear to be a git repository") || + strings.Contains(outputStr, "Could not read from remote repository") { + fmt.Printf(" Note: Remote repository %s does not exist - creating it first would be needed\n", remoteName) + fmt.Printf(" Skipping push to %s (repository must be created manually)\n", remoteName) + continue + } // Check if it's because the branch doesn't exist on the remote - if strings.Contains(string(output), "error: src refspec") { + if strings.Contains(outputStr, "error: src refspec") { fmt.Printf(" Creating new branch on %s\n", remoteName) // Try again with -u flag to set upstream cmd = exec.Command("git", "push", "-u", remoteName, branch) @@ -251,7 +310,7 @@ func (s *Syncer) syncBranch(branch string, remotes map[string]*config.Organizati return fmt.Errorf("failed to push to %s: %w", remoteName, err) } } else { - return fmt.Errorf("failed to push to %s: %w\n%s", remoteName, err, string(output)) + return fmt.Errorf("failed to push to %s: %w\n%s", remoteName, err, outputStr) } } } diff --git a/test/run_integration_tests.sh b/test/run_integration_tests.sh new file mode 100755 index 0000000..b0c18a8 --- /dev/null +++ b/test/run_integration_tests.sh @@ -0,0 +1,229 @@ +#!/bin/bash + +# Integration test script for gitsyncer +# This script runs all tests in sequence and reports results + +set -e + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +NC='\033[0m' # No Color + +# Get script directory +TEST_DIR="$(cd "$(dirname "$0")" && pwd)" +PROJECT_ROOT="$(dirname "$TEST_DIR")" + +# Function to print test headers +print_test() { + echo -e "\n${YELLOW}=== $1 ===${NC}" +} + +# Function to print success +print_success() { + echo -e "${GREEN}✓ $1${NC}" +} + +# Function to print failure +print_failure() { + echo -e "${RED}✗ $1${NC}" +} + +# Start testing +echo "GitSyncer Integration Tests" +echo "============================" + +# Clean up any previous test artifacts +print_test "Cleaning up previous test artifacts" +cd "$TEST_DIR" +rm -rf repos work work-* test-config.json +print_success "Cleanup complete" + +# Build the project +print_test "Building gitsyncer" +cd "$PROJECT_ROOT" +if go build -o gitsyncer ./cmd/gitsyncer; then + print_success "Build successful" +else + print_failure "Build failed" + exit 1 +fi + +# Test 1: Version flag +print_test "Test 1: Version flag" +if ./gitsyncer --version | grep -q "gitsyncer version"; then + print_success "Version flag works" +else + print_failure "Version flag failed" + exit 1 +fi + +# Test 2: Setup test repositories +print_test "Test 2: Setting up test repositories" +cd "$TEST_DIR" +if ./setup_test_repos.sh > /dev/null 2>&1; then + print_success "Test repositories created" + echo " - org1: Has main, develop, and feature/test-feature branches" + echo " - org2: Has only main branch" +else + print_failure "Failed to setup test repositories" + exit 1 +fi + +# Test 3: List organizations +print_test "Test 3: List organizations" +cd "$PROJECT_ROOT" +if ./gitsyncer --config test/test-config.json --list-orgs | grep -q "org1"; then + print_success "Organizations listed successfully" +else + print_failure "Failed to list organizations" + exit 1 +fi + +# Test 4: Initial sync +print_test "Test 4: Initial repository sync" +rm -rf test/work +if ./gitsyncer --config test/test-config.json --sync test-repo --work-dir test/work > /dev/null 2>&1; then + print_success "Initial sync completed" + + # Verify all branches are synced + cd "$TEST_DIR" + ORG2_BRANCHES=$(git ls-remote file://$(pwd)/repos/org2/test-repo.git | grep refs/heads | wc -l) + if [ "$ORG2_BRANCHES" -eq "3" ]; then + print_success "All branches synced to org2 (found $ORG2_BRANCHES branches)" + else + print_failure "Branch sync failed - expected 3 branches, found $ORG2_BRANCHES" + exit 1 + fi +else + print_failure "Initial sync failed" + exit 1 +fi + +# Test 5: Sync with no changes (idempotent) +print_test "Test 5: Idempotent sync (no changes)" +cd "$PROJECT_ROOT" +rm -rf test/work2 +if ./gitsyncer --config test/test-config.json --sync test-repo --work-dir test/work2 > /dev/null 2>&1; then + print_success "Idempotent sync successful" +else + print_failure "Idempotent sync failed" + exit 1 +fi + +# Test 6: Create changes and sync +print_test "Test 6: Sync with new changes" +cd "$TEST_DIR" +# Create a change in org1 +WORK_DIR="$TEST_DIR/repos/change-work" +rm -rf "$WORK_DIR" +mkdir -p "$WORK_DIR" +cd "$WORK_DIR" +git clone -q file://"$TEST_DIR"/repos/org1/test-repo.git +cd test-repo +echo "New feature added" > feature2.txt +git add feature2.txt +git commit -q -m "Add feature2.txt" +git push -q origin main + +# Run sync +cd "$PROJECT_ROOT" +rm -rf test/work3 +if ./gitsyncer --config test/test-config.json --sync test-repo --work-dir test/work3 > /dev/null 2>&1; then + print_success "Sync with changes successful" + + # Verify change is in org2 + cd "$TEST_DIR" + if git ls-remote file://$(pwd)/repos/org2/test-repo.git HEAD | grep -q "$(git ls-remote file://$(pwd)/repos/org1/test-repo.git HEAD | cut -f1)"; then + print_success "Changes propagated to org2" + else + print_failure "Changes not propagated to org2" + exit 1 + fi +else + print_failure "Sync with changes failed" + exit 1 +fi + +# Clean up work directory +rm -rf "$WORK_DIR" + +# Test 7: Conflict detection +print_test "Test 7: Merge conflict detection" +cd "$TEST_DIR" +if ./test_conflict.sh > /dev/null 2>&1; then + print_success "Conflicting changes created" +else + print_failure "Failed to create conflicting changes" + exit 1 +fi + +# Try to sync with conflicts (should fail) +cd "$PROJECT_ROOT" +if ./gitsyncer --config test/test-config.json --sync test-repo --work-dir test/work-conflict > /dev/null 2>&1; then + print_failure "Sync should have failed with conflicts but didn't" + exit 1 +else + print_success "Conflict detection working - sync correctly failed" +fi + +# Test 8: Missing config file +print_test "Test 8: Missing config file handling" +if ./gitsyncer --config test/nonexistent.json --sync test-repo 2>&1 | grep -q "Failed to load configuration"; then + print_success "Missing config file handled correctly" +else + print_failure "Missing config file not handled properly" + exit 1 +fi + +# Test 9: Empty organization list +print_test "Test 9: Invalid configuration handling" +cd "$TEST_DIR" +echo '{"organizations": []}' > empty-config.json +cd "$PROJECT_ROOT" +if ./gitsyncer --config test/empty-config.json --sync test-repo 2>&1 | grep -q "no organizations configured"; then + print_success "Empty organization list handled correctly" +else + print_failure "Empty organization list not handled properly" + exit 1 +fi + +# Test 10: Multiple repository configuration +print_test "Test 10: Multiple repository sync" +cd "$TEST_DIR" +./test_multi_repo.sh > /dev/null 2>&1 +cd "$PROJECT_ROOT" + +# Test list-repos +if ./gitsyncer --config test/multi-repo-config.json --list-repos | grep -q "repo1"; then + print_success "Repository listing works" +else + print_failure "Failed to list repositories" + exit 1 +fi + +# Test sync-all +if ./gitsyncer --config test/multi-repo-config.json --sync-all --work-dir test/work-multi > /dev/null 2>&1; then + print_success "Multiple repository sync completed" +else + print_failure "Multiple repository sync failed" + exit 1 +fi + +# Cleanup test artifacts +print_test "Cleaning up test artifacts" +cd "$TEST_DIR" +rm -rf repos work work-* empty-config.json multi-repo-config.json +print_success "Cleanup complete" + +# Summary +echo -e "\n${GREEN}=== All tests passed! ===${NC}" +echo "GitSyncer is working correctly and can:" +echo " ✓ Sync repositories between multiple organizations" +echo " ✓ Handle all branches automatically" +echo " ✓ Merge changes from all remotes" +echo " ✓ Detect and report merge conflicts" +echo " ✓ Handle configuration errors gracefully" +echo " ✓ Sync multiple repositories with --sync-all" +echo " ✓ Handle missing remote repositories gracefully" \ No newline at end of file diff --git a/test/test_multi_repo.sh b/test/test_multi_repo.sh new file mode 100755 index 0000000..d0b3b8a --- /dev/null +++ b/test/test_multi_repo.sh @@ -0,0 +1,102 @@ +#!/bin/bash + +# Test script for multiple repository sync functionality +set -e + +echo "Setting up test for multiple repository sync..." + +TEST_DIR="$(cd "$(dirname "$0")" && pwd)" +REPOS_DIR="$TEST_DIR/repos" +mkdir -p "$REPOS_DIR" + +# Create two bare repositories in each organization +ORG1_DIR="$REPOS_DIR/org1" +ORG2_DIR="$REPOS_DIR/org2" + +# Clean up if they exist +rm -rf "$ORG1_DIR" "$ORG2_DIR" +mkdir -p "$ORG1_DIR" "$ORG2_DIR" + +# Create repo1 and repo2 in org1 +echo "Creating repositories in org1..." +cd "$ORG1_DIR" +git init --bare repo1.git +git init --bare repo2.git + +# Create only repo1 in org2 (repo2 will be synced) +echo "Creating repository in org2..." +cd "$ORG2_DIR" +git init --bare repo1.git + +# Add initial content to repo1 in org1 +WORK_DIR="$REPOS_DIR/work" +rm -rf "$WORK_DIR" +mkdir -p "$WORK_DIR" + +echo "Adding content to repo1..." +cd "$WORK_DIR" +git clone "$ORG1_DIR/repo1.git" +cd repo1 +echo "# Repo 1" > README.md +echo "This is repository 1" >> README.md +git add README.md +git commit -m "Initial commit for repo1" +git push origin main + +# Add initial content to repo2 in org1 +echo "Adding content to repo2..." +cd "$WORK_DIR" +git clone "$ORG1_DIR/repo2.git" +cd repo2 +echo "# Repo 2" > README.md +echo "This is repository 2" >> README.md +mkdir src +echo "console.log('Hello from repo2');" > src/index.js +git add . +git commit -m "Initial commit for repo2" +git push origin main + +# Create feature branch in repo2 +git checkout -b feature/awesome +echo "Awesome feature" > feature.txt +git add feature.txt +git commit -m "Add awesome feature" +git push origin feature/awesome + +# Clean up work directory +cd "$TEST_DIR" +rm -rf "$WORK_DIR" + +# Create test config with repositories list +echo "Creating test configuration with repositories..." +cat > "$TEST_DIR/multi-repo-config.json" << EOF +{ + "organizations": [ + { + "host": "file://$ORG1_DIR", + "name": "" + }, + { + "host": "file://$ORG2_DIR", + "name": "" + } + ], + "repositories": [ + "repo1", + "repo2" + ] +} +EOF + +echo "" +echo "Test setup complete!" +echo "" +echo "Repository structure:" +echo "- org1/repo1.git: Has initial content" +echo "- org1/repo2.git: Has main and feature/awesome branches" +echo "- org2/repo1.git: Empty (will receive updates)" +echo "- org2/repo2.git: Does not exist (will be created)" +echo "" +echo "Test with:" +echo " ./gitsyncer --config test/multi-repo-config.json --list-repos" +echo " ./gitsyncer --config test/multi-repo-config.json --sync-all" \ No newline at end of file -- cgit v1.2.3