From c69715f2753adbd59b4f01e09f5771bb9c37a517 Mon Sep 17 00:00:00 2001 From: Florian Beisel Date: Thu, 21 Mar 2024 01:39:50 +0100 Subject: [PATCH] Initial commit --- go.mod | 27 +++++++ go.sum | 69 +++++++++++++++++ main.go | 233 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 329 insertions(+) create mode 100644 go.mod create mode 100644 go.sum create mode 100644 main.go diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..82234c5 --- /dev/null +++ b/go.mod @@ -0,0 +1,27 @@ +module git.beisel.it/florian/go-logrotate + +go 1.21.1 + +require github.com/spf13/viper v1.18.2 + +require ( + github.com/fsnotify/fsnotify v1.7.0 // indirect + github.com/hashicorp/hcl v1.0.0 // indirect + github.com/magiconair/properties v1.8.7 // indirect + github.com/mitchellh/mapstructure v1.5.0 // indirect + github.com/pelletier/go-toml/v2 v2.1.0 // indirect + github.com/sagikazarmark/locafero v0.4.0 // indirect + github.com/sagikazarmark/slog-shim v0.1.0 // indirect + github.com/sourcegraph/conc v0.3.0 // indirect + github.com/spf13/afero v1.11.0 // indirect + github.com/spf13/cast v1.6.0 // indirect + github.com/spf13/pflag v1.0.5 // indirect + github.com/subosito/gotenv v1.6.0 // indirect + go.uber.org/atomic v1.9.0 // indirect + go.uber.org/multierr v1.9.0 // indirect + golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect + golang.org/x/sys v0.15.0 // indirect + golang.org/x/text v0.14.0 // indirect + gopkg.in/ini.v1 v1.67.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..eaf76d3 --- /dev/null +++ b/go.sum @@ -0,0 +1,69 @@ +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= +github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= +github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= +github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= +github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= +github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= +github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= +github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= +github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6keLGt6kNQ= +github.com/sagikazarmark/locafero v0.4.0/go.mod h1:Pe1W6UlPYUk/+wc/6KFhbORCfqzgYEpgQ3O5fPuL3H4= +github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE= +github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ= +github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= +github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= +github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8= +github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY= +github.com/spf13/cast v1.6.0 h1:GEiTHELF+vaR5dhz3VqZfFSzZjYbgeKDpBxQVS4GYJ0= +github.com/spf13/cast v1.6.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.18.2 h1:LUXCnvUvSM6FXAsj6nnfc8Q2tp1dIgUfY9Kc8GsSOiQ= +github.com/spf13/viper v1.18.2/go.mod h1:EKmWIqdnk5lOcmR72yw6hS+8OPYcwD0jteitLMVB+yk= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= +github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= +go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= +go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI= +go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ= +golang.org/x/exp v0.0.0-20230905200255-921286631fa9 h1:GoHiUyI/Tp2nVkLI2mCxVkOjsbSXD66ic0XW0js0R9g= +golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqRShND87VCbxmc6bL1Yd2oYrm6k= +golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= +golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= +gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/main.go b/main.go new file mode 100644 index 0000000..97cc985 --- /dev/null +++ b/main.go @@ -0,0 +1,233 @@ +package main + +import ( + "compress/gzip" + "flag" + "fmt" + "io" + "os" + "strconv" + "strings" + "time" + + "github.com/spf13/viper" +) + +// GlobalSettings represents global configuration settings +type GlobalSettings struct { + Compress bool `mapstructure:"compress"` + CompressFormat string `mapstructure:"compressFormat"` + CreateNew bool `mapstructure:"createNew"` + CreateMode string `mapstructure:"createMode"` +} + +// LogConfig represents configuration for a specific log file +type LogConfig struct { + Path string `mapstructure:"path"` + RotateOnSize string `mapstructure:"rotateOnSize"` // This will be parsed to get the size in bytes + RetentionCount int `mapstructure:"retentionCount"` + RotateOnAge time.Duration `mapstructure:"rotateOnAge"` // Assume the input is compatible with time.ParseDuration + Compress bool `mapstructure:"compress"` // Optional: per-log override +} + +// Configuration holds the entire configuration +type Configuration struct { + Settings GlobalSettings `mapstructure:"settings"` + Logs []LogConfig `mapstructure:"logs"` +} + +// ConvertSizeToBytes converts size strings like "10MB" to bytes +func ConvertSizeToBytes(sizeStr string) (int64, error) { + sizeStr = strings.ToUpper(sizeStr) + multiplier := int64(1) + + if strings.HasSuffix(sizeStr, "KB") { + multiplier = 1024 + sizeStr = strings.TrimSuffix(sizeStr, "KB") + } else if strings.HasSuffix(sizeStr, "MB") { + multiplier = 1024 * 1024 + sizeStr = strings.TrimSuffix(sizeStr, "MB") + } else if strings.HasSuffix(sizeStr, "GB") { + multiplier = 1024 * 1024 * 1024 + sizeStr = strings.TrimSuffix(sizeStr, "GB") + } + + size, err := strconv.ParseInt(sizeStr, 10, 64) + if err != nil { + return 0, err + } + + return size * multiplier, nil +} + +func compressFile(sourcePath, destPath string) error { + inputFile, err := os.Open(sourcePath) + if err != nil { + return err + } + defer inputFile.Close() // Ensure the file is closed as soon as the function returns + + outputFile, err := os.Create(destPath) + if err != nil { + return err + } + defer outputFile.Close() // Ensure the file is closed as soon as the function returns + + gzWriter := gzip.NewWriter(outputFile) + defer gzWriter.Close() // Ensure the gzip writer is closed + + _, err = io.Copy(gzWriter, inputFile) + if err != nil { + return err + } + + // Ensure all writes are flushed and the writer is closed before deleting the source file + if err := gzWriter.Close(); err != nil { + return err + } + if err := outputFile.Close(); err != nil { + return err + } + if err := inputFile.Close(); err != nil { + return err + } + + // After confirming the file has been compressed and closed, it's safe to remove the source file + return os.Remove(sourcePath) +} + +func shouldRotateBasedOnAge(filePath string, rotateAge time.Duration) (bool, error) { + fileInfo, err := os.Stat(filePath) + if err != nil { + return false, err + } + + // Calculate the file's age by subtracting its modification time from the current time + fileAge := time.Since(fileInfo.ModTime()) + + // Return true if the file's age exceeds the rotateAge threshold + return fileAge >= rotateAge, nil +} + +// rotateFile rotates the specified log file according to the retention policy +func rotateFile(config LogConfig, verbose bool) error { + // First, check if the current log file exceeds the size threshold + rotateSize, err := ConvertSizeToBytes(config.RotateOnSize) + if err != nil { + return fmt.Errorf("error converting rotation size: %w", err) + } + + fileInfo, err := os.Stat(config.Path) + if err != nil { + return fmt.Errorf("error stating current log file: %w", err) + } + + shouldRotate := fileInfo.Size() >= rotateSize + + // Determine if the file needs rotation based on age, if rotateOnAge is specified + if config.RotateOnAge > 0 { + rotateBasedOnAge, err := shouldRotateBasedOnAge(config.Path, config.RotateOnAge) + if err != nil { + return fmt.Errorf("error checking if log file should be rotated based on age: %w", err) + } + shouldRotate = shouldRotate || rotateBasedOnAge + } + + if shouldRotate { + + // File needs rotation; handle retention for existing rotated files + compressExtension := ".gz" + shouldCompress := config.Compress || (!config.Compress && viper.GetBool("settings.compress")) + + // Rotate existing files, considering compression + for i := config.RetentionCount; i > 0; i-- { + currentFile := fmt.Sprintf("%s.%d", config.Path, i) + nextFileNumber := i + 1 + nextFile := fmt.Sprintf("%s.%d", config.Path, nextFileNumber) + + if shouldCompress { + currentFile += compressExtension + nextFile += compressExtension + } + + if _, err := os.Stat(currentFile); !os.IsNotExist(err) { + if i == config.RetentionCount { + // Remove the oldest file + os.Remove(currentFile) + } else { + // Rename (rotate) the file to the next higher number + os.Rename(currentFile, nextFile) + } + } + } + + // Rotate the current log file + rotatedFileName := config.Path + ".1" + if shouldCompress { + // If compression is enabled, compress the file + compressedFileName := rotatedFileName + compressExtension + if err := compressFile(config.Path, compressedFileName); err != nil { + return fmt.Errorf("failed to compress log file: %w", err) + } + } else { + // If not compressing, simply rename the current log file + if err := os.Rename(config.Path, rotatedFileName); err != nil { + return fmt.Errorf("failed to rotate log file: %w", err) + } + } + + // If createNew is true, create a new log file with the original log file's name + if viper.GetBool("settings.createNew") { + mode, _ := strconv.ParseUint(viper.GetString("settings.createMode"), 0, 32) + newFile, err := os.OpenFile(config.Path, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, os.FileMode(mode)) + if err != nil { + return fmt.Errorf("failed to create new log file: %w", err) + } + if err := newFile.Close(); err != nil { + return fmt.Errorf("failed to close new log file: %w", err) + } + } + } + return nil +} + +func main() { + var verbose bool + flag.BoolVar(&verbose, "verbose", false, "enable verbose output") + flag.Parse() + + viper.SetConfigName("config.yaml") + viper.SetConfigType("yaml") + viper.AddConfigPath(".") + + var config Configuration + + if err := viper.ReadInConfig(); err != nil { + fmt.Printf("Error reading config file, %s", err) + os.Exit(1) + } + + if err := viper.Unmarshal(&config); err != nil { + fmt.Printf("Unable to decode into struct, %s", err) + os.Exit(1) + } + + if verbose { + fmt.Printf("Global Compress: %t\n", config.Settings.Compress) + for _, log := range config.Logs { + fmt.Printf("Log Path: %s, RotateOnSize: %s, RotateOnAge: %s\n", log.Path, log.RotateOnSize, log.RotateOnAge) + } + } + + // Process each log file according to its configuration + for _, logConfig := range config.Logs { + if err := rotateFile(logConfig, verbose); err != nil { + fmt.Printf("Error rotating file %s: %s\n", logConfig.Path, err) + continue // Proceed to the next log file instead of stopping + } + + if verbose { + fmt.Printf("Processed log file: %s\n", logConfig.Path) + } + } +}