Commit 5ddde196 authored by glutamatt's avatar glutamatt Committed by Krasi Georgiev
Browse files

tune the "Wal segment size" with a flag (#5029)

Add WALSegmentSize as an option, and the corresponding flag "storage.tsdb.wal-segment-size" to tune the max size of wal segment files.

The addressed base problem is to reduce the disk space used by wal segment files : on a raspberry pi, for instance, we often want to reduce write load of the sd card, then, the wal directory is mounted on a memory (space limited) partition.

the default value of the segment max file size, pushed the size of directory to 128 MB for each segment , which is too much ram consumption on a rasp.

the initial discussion is at https://github.com/prometheus/tsdb/pull/450
parent 121603c4
......@@ -164,6 +164,10 @@ func main() {
"Maximum duration compacted blocks may span. For use in testing. (Defaults to 10% of the retention period).").
Hidden().PlaceHolder("<duration>").SetValue(&cfg.tsdb.MaxBlockDuration)
a.Flag("storage.tsdb.wal-segment-size",
"Size at which to split the tsdb WAL segment files (e.g. 100MB)").
Hidden().PlaceHolder("<bytes>").BytesVar(&cfg.tsdb.WALSegmentSize)
a.Flag("storage.tsdb.retention", "How long to retain samples in storage.").
Default("15d").SetValue(&cfg.tsdb.Retention)
......@@ -560,6 +564,11 @@ func main() {
g.Add(
func() error {
level.Info(logger).Log("msg", "Starting TSDB ...")
if cfg.tsdb.WALSegmentSize != 0 {
if cfg.tsdb.WALSegmentSize < 10*1024*1024 || cfg.tsdb.WALSegmentSize > 256*1024*1024 {
return errors.New("flag 'storage.tsdb.wal-segment-size' must be set between 10MB and 256MB")
}
}
db, err := tsdb.Open(
cfg.localStoragePath,
log.With(logger, "component", "tsdb"),
......
......@@ -247,3 +247,36 @@ func TestSendAlerts(t *testing.T) {
})
}
}
func TestWALSegmentSizeBounds(t *testing.T) {
if testing.Short() {
t.Skip("skipping test in short mode.")
}
for size, expectedExitStatus := range map[string]int{"9MB": 1, "257MB": 1, "10": 2, "1GB": 1, "12MB": 0} {
prom := exec.Command(promPath, "--storage.tsdb.wal-segment-size="+size, "--config.file="+promConfig)
err := prom.Start()
testutil.Ok(t, err)
if expectedExitStatus == 0 {
done := make(chan error, 1)
go func() { done <- prom.Wait() }()
select {
case err := <-done:
t.Errorf("prometheus should be still running: %v", err)
case <-time.After(5 * time.Second):
prom.Process.Signal(os.Interrupt)
}
continue
}
err = prom.Wait()
testutil.NotOk(t, err, "")
if exitError, ok := err.(*exec.ExitError); ok {
status := exitError.Sys().(syscall.WaitStatus)
testutil.Equals(t, expectedExitStatus, status.ExitStatus())
} else {
t.Errorf("unable to retrieve the exit status for prometheus: %v", err)
}
}
}
......@@ -5,6 +5,7 @@ require (
github.com/Azure/go-autorest v10.8.1+incompatible
github.com/StackExchange/wmi v0.0.0-20180725035823-b12b22c5341f // indirect
github.com/VividCortex/ewma v1.1.1 // indirect
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf
github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da // indirect
github.com/aws/aws-sdk-go v0.0.0-20180507225419-00862f899353
github.com/biogo/store v0.0.0-20160505134755-913427a1d5e8 // indirect
......@@ -88,7 +89,7 @@ require (
github.com/prometheus/client_golang v0.9.1
github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910
github.com/prometheus/common v0.0.0-20181119215939-b36ad289a3ea
github.com/prometheus/tsdb v0.3.1
github.com/prometheus/tsdb v0.3.2-0.20181219094047-6d489a1004dc
github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a // indirect
github.com/rlmcpherson/s3gof3r v0.5.0 // indirect
github.com/rubyist/circuitbreaker v2.2.1+incompatible // indirect
......
......@@ -19,6 +19,7 @@ import (
"time"
"unsafe"
"github.com/alecthomas/units"
"github.com/go-kit/kit/log"
"github.com/pkg/errors"
"github.com/prometheus/client_golang/prometheus"
......@@ -117,6 +118,9 @@ type Options struct {
// The maximum timestamp range of compacted blocks.
MaxBlockDuration model.Duration
// The maximum size of each WAL segment file.
WALSegmentSize units.Base2Bytes
// Duration for how long to retain data.
Retention model.Duration
......@@ -182,6 +186,7 @@ func Open(path string, l log.Logger, r prometheus.Registerer, opts *Options) (*t
db, err := tsdb.Open(path, l, r, &tsdb.Options{
WALFlushInterval: 10 * time.Second,
WALSegmentSize: int(opts.WALSegmentSize),
RetentionDuration: uint64(time.Duration(opts.Retention).Seconds() * 1000),
BlockRanges: rngs,
NoLockfile: opts.NoLockfile,
......
## master / unreleased
- [CHANGE] New `WALSegmentSize` option to override the `DefaultOptions.WALSegmentSize`. Added to allow using smaller wal files. For example using tmpfs on a RPI to minimise the SD card wear out from the constant WAL writes. As part of this change the `DefaultOptions.WALSegmentSize` constant was also exposed.
## 0.3.1
- [BUGFIX] Fixed most windows test and some actual bugs for unclosed file readers.
## 0.3.0
- [CHANGE] `LastCheckpoint()` used to return just the segment name and now it returns the full relative path.
- [CHANGE] `NewSegmentsRangeReader()` can now read over miltiple wal ranges by using the new `SegmentRange{}` struct.
- [CHANGE] `CorruptionErr{}` now also exposes the Segment `Dir` which is added when displaying any errors.
......
......@@ -45,6 +45,7 @@ import (
// millisecond precision timestamps.
var DefaultOptions = &Options{
WALFlushInterval: 5 * time.Second,
WALSegmentSize: wal.DefaultSegmentSize,
RetentionDuration: 15 * 24 * 60 * 60 * 1000, // 15 days in milliseconds
BlockRanges: ExponentialBlockRanges(int64(2*time.Hour)/1e6, 3, 5),
NoLockfile: false,
......@@ -55,6 +56,9 @@ type Options struct {
// The interval at which the write ahead log is flushed to disk.
WALFlushInterval time.Duration
// Segments (wal files) max size
WALSegmentSize int
// Duration of persisted data to keep.
RetentionDuration uint64
......@@ -263,7 +267,11 @@ func Open(dir string, l log.Logger, r prometheus.Registerer, opts *Options) (db
return nil, errors.Wrap(err, "create leveled compactor")
}
wlog, err := wal.New(l, r, filepath.Join(dir, "wal"))
segmentSize := wal.DefaultSegmentSize
if opts.WALSegmentSize > 0 {
segmentSize = opts.WALSegmentSize
}
wlog, err := wal.NewSize(l, r, filepath.Join(dir, "wal"), segmentSize)
if err != nil {
return nil, err
}
......
......@@ -89,6 +89,7 @@ type headMetrics struct {
maxTime prometheus.GaugeFunc
samplesAppended prometheus.Counter
walTruncateDuration prometheus.Summary
walCorruptionsTotal prometheus.Counter
headTruncateFail prometheus.Counter
headTruncateTotal prometheus.Counter
checkpointDeleteFail prometheus.Counter
......@@ -152,6 +153,10 @@ func newHeadMetrics(h *Head, r prometheus.Registerer) *headMetrics {
Name: "prometheus_tsdb_wal_truncate_duration_seconds",
Help: "Duration of WAL truncation.",
})
m.walCorruptionsTotal = prometheus.NewCounter(prometheus.CounterOpts{
Name: "prometheus_tsdb_wal_corruptions_total",
Help: "Total number of WAL corruptions.",
})
m.samplesAppended = prometheus.NewCounter(prometheus.CounterOpts{
Name: "prometheus_tsdb_head_samples_appended_total",
Help: "Total number of appended samples.",
......@@ -195,6 +200,7 @@ func newHeadMetrics(h *Head, r prometheus.Registerer) *headMetrics {
m.maxTime,
m.gcDuration,
m.walTruncateDuration,
m.walCorruptionsTotal,
m.samplesAppended,
m.headTruncateFail,
m.headTruncateTotal,
......@@ -480,10 +486,10 @@ func (h *Head) Init(minValidTime int64) error {
return nil
}
level.Warn(h.logger).Log("msg", "encountered WAL error, attempting repair", "err", err)
h.metrics.walCorruptionsTotal.Inc()
if err := h.wal.Repair(err); err != nil {
return errors.Wrap(err, "repair corrupted WAL")
}
return nil
}
......@@ -500,7 +506,7 @@ func (h *Head) Truncate(mint int64) (err error) {
return nil
}
atomic.StoreInt64(&h.minTime, mint)
h.minValidTime = mint
atomic.StoreInt64(&h.minValidTime, mint)
// Ensure that max time is at least as high as min time.
for h.MaxTime() < mint {
......@@ -656,7 +662,7 @@ func (h *Head) appender() *headAppender {
head: h,
// Set the minimum valid time to whichever is greater the head min valid time or the compaciton window.
// This ensures that no samples will be added within the compaction window to avoid races.
minValidTime: max(h.minValidTime, h.MaxTime()-h.chunkRange/2),
minValidTime: max(atomic.LoadInt64(&h.minValidTime), h.MaxTime()-h.chunkRange/2),
mint: math.MaxInt64,
maxt: math.MinInt64,
samples: h.getAppendBuffer(),
......
......@@ -35,7 +35,7 @@ import (
)
const (
defaultSegmentSize = 128 * 1024 * 1024 // 128 MB
DefaultSegmentSize = 128 * 1024 * 1024 // 128 MB
pageSize = 32 * 1024 // 32KB
recordHeaderSize = 7
)
......@@ -174,7 +174,7 @@ type WAL struct {
// New returns a new WAL over the given directory.
func New(logger log.Logger, reg prometheus.Registerer, dir string) (*WAL, error) {
return NewSize(logger, reg, dir, defaultSegmentSize)
return NewSize(logger, reg, dir, DefaultSegmentSize)
}
// NewSize returns a new WAL over the given directory.
......
......@@ -187,7 +187,7 @@ github.com/prometheus/procfs
github.com/prometheus/procfs/nfs
github.com/prometheus/procfs/xfs
github.com/prometheus/procfs/internal/util
# github.com/prometheus/tsdb v0.3.1
# github.com/prometheus/tsdb v0.3.2-0.20181219094047-6d489a1004dc
github.com/prometheus/tsdb
github.com/prometheus/tsdb/labels
github.com/prometheus/tsdb/chunkenc
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment