137 lines
3.8 KiB
Go
137 lines
3.8 KiB
Go
package spool
|
|
|
|
import (
|
|
"encoding/json"
|
|
"fmt"
|
|
"os"
|
|
"path/filepath"
|
|
"sort"
|
|
"strings"
|
|
|
|
"browser.local/run/protocol"
|
|
)
|
|
|
|
type LogSpool struct {
|
|
dir string
|
|
}
|
|
|
|
func NewLogSpool(dir string) (LogSpool, error) {
|
|
if strings.TrimSpace(dir) == "" {
|
|
return LogSpool{}, fmt.Errorf("spool directory is required")
|
|
}
|
|
logDir := filepath.Join(dir, "logs")
|
|
if err := os.MkdirAll(logDir, 0o755); err != nil {
|
|
return LogSpool{}, fmt.Errorf("create log spool: %w", err)
|
|
}
|
|
return LogSpool{dir: logDir}, nil
|
|
}
|
|
|
|
func (spool LogSpool) Enqueue(batch protocol.LogBatchIngestRequest) error {
|
|
path := spool.batchPath(batch)
|
|
tmp := path + ".tmp"
|
|
file, err := os.OpenFile(tmp, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o644)
|
|
if err != nil {
|
|
return fmt.Errorf("open log spool segment: %w", err)
|
|
}
|
|
encodeErr := json.NewEncoder(file).Encode(batch)
|
|
closeErr := file.Close()
|
|
if encodeErr != nil {
|
|
_ = os.Remove(tmp)
|
|
return fmt.Errorf("encode log spool segment: %w", encodeErr)
|
|
}
|
|
if closeErr != nil {
|
|
_ = os.Remove(tmp)
|
|
return fmt.Errorf("close log spool segment: %w", closeErr)
|
|
}
|
|
if err := os.Rename(tmp, path); err != nil {
|
|
_ = os.Remove(tmp)
|
|
return fmt.Errorf("commit log spool segment: %w", err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (spool LogSpool) Pending() ([]protocol.LogBatchIngestRequest, error) {
|
|
entries, err := os.ReadDir(spool.dir)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("read log spool: %w", err)
|
|
}
|
|
paths := make([]string, 0, len(entries))
|
|
for _, entry := range entries {
|
|
if entry.IsDir() || !strings.HasSuffix(entry.Name(), ".json") {
|
|
continue
|
|
}
|
|
paths = append(paths, filepath.Join(spool.dir, entry.Name()))
|
|
}
|
|
sort.Strings(paths)
|
|
batches := make([]protocol.LogBatchIngestRequest, 0, len(paths))
|
|
for _, path := range paths {
|
|
file, err := os.Open(path)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("open log spool segment: %w", err)
|
|
}
|
|
var batch protocol.LogBatchIngestRequest
|
|
decodeErr := json.NewDecoder(file).Decode(&batch)
|
|
closeErr := file.Close()
|
|
if decodeErr != nil {
|
|
return nil, fmt.Errorf("decode log spool segment: %w", decodeErr)
|
|
}
|
|
if closeErr != nil {
|
|
return nil, fmt.Errorf("close log spool segment: %w", closeErr)
|
|
}
|
|
batches = append(batches, batch)
|
|
}
|
|
return batches, nil
|
|
}
|
|
|
|
func (spool LogSpool) Ack(response protocol.LogBatchIngestResponse) error {
|
|
entries, err := os.ReadDir(spool.dir)
|
|
if err != nil {
|
|
return fmt.Errorf("read log spool: %w", err)
|
|
}
|
|
for _, entry := range entries {
|
|
if entry.IsDir() || !strings.HasSuffix(entry.Name(), ".json") {
|
|
continue
|
|
}
|
|
path := filepath.Join(spool.dir, entry.Name())
|
|
file, err := os.Open(path)
|
|
if err != nil {
|
|
return fmt.Errorf("open log spool segment: %w", err)
|
|
}
|
|
var batch protocol.LogBatchIngestRequest
|
|
decodeErr := json.NewDecoder(file).Decode(&batch)
|
|
closeErr := file.Close()
|
|
if decodeErr != nil {
|
|
return fmt.Errorf("decode log spool segment: %w", decodeErr)
|
|
}
|
|
if closeErr != nil {
|
|
return fmt.Errorf("close log spool segment: %w", closeErr)
|
|
}
|
|
if batch.LogStreamID == response.LogStreamID && batch.FirstSeq >= response.AcceptedFrom && batch.LastSeq <= response.AcceptedTo {
|
|
if err := os.Remove(path); err != nil {
|
|
return fmt.Errorf("remove acknowledged log spool segment: %w", err)
|
|
}
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (spool LogSpool) batchPath(batch protocol.LogBatchIngestRequest) string {
|
|
streamID := sanitizeSegmentName(batch.LogStreamID)
|
|
return filepath.Join(spool.dir, fmt.Sprintf("%s-%020d-%020d.json", streamID, batch.FirstSeq, batch.LastSeq))
|
|
}
|
|
|
|
func sanitizeSegmentName(value string) string {
|
|
var builder strings.Builder
|
|
for _, r := range value {
|
|
if r >= 'a' && r <= 'z' || r >= 'A' && r <= 'Z' || r >= '0' && r <= '9' || r == '-' || r == '_' || r == '.' {
|
|
builder.WriteRune(r)
|
|
continue
|
|
}
|
|
builder.WriteByte('_')
|
|
}
|
|
if builder.Len() == 0 {
|
|
return "stream"
|
|
}
|
|
return builder.String()
|
|
}
|