add metra, a scraper

3 files changed, 228 insertions, 2 deletions

diff --git a/Makefile b/Makefile
index e80b676..2f8f289 100644
--- a/Makefile
+++ b/Makefile
@@ -1,9 +1,12 @@
 CFLAGS=-g -O2 -Wall -Wno-switch -Wextra -Wwrite-strings
 LDLIBS=-lzip
 
-all: mico-store mico-dump
+all: mico-store mico-dump metra/metra
+
+metra/metra: metra/metra.go metra/go.mod
+	cd metra && go build -v
 
 clean: FRC
-	rm -f mico-store
+	rm -f mico-store mico-dump metra/metra
 
 FRC:
diff --git a/metra/go.mod b/metra/go.mod
new file mode 100644
index 0000000..222ee0e
--- /dev/null
+++ b/metra/go.mod
@@ -0,0 +1,3 @@
+module github.com/leahneukirchen/mico/metra
+
+go 1.21.1
diff --git a/metra/metra.go b/metra/metra.go
new file mode 100644
index 0000000..bab66a0
--- /dev/null
+++ b/metra/metra.go
@@ -0,0 +1,220 @@
+package main
+
+import (
+	"bufio"
+	"fmt"
+	"io"
+	"log"
+	"net/http"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"regexp"
+	"strconv"
+	"strings"
+	"time"
+)
+
+type Filter struct {
+	keep    bool
+	pattern *regexp.Regexp
+}
+
+const userAgent = "metra/0.1"
+
+var filters []Filter
+
+func responseHandler(body io.ReadCloser, timestamp int64) {
+	scanner := bufio.NewScanner(body)
+	defer body.Close()
+
+	for scanner.Scan() {
+		line := scanner.Text()
+
+		if line == "" || strings.HasPrefix(line, "#") {
+			continue
+		}
+
+		// XXX detect if a timestamp is there and forward it?
+		// XXX normalize label order
+		// XXX normalize spacing
+
+		// XXX add labels from config file, targets?
+
+		sp := strings.LastIndexByte(line, ' ')
+		if sp > 0 {
+			name := line[:sp]
+			value := line[sp+1:]
+
+			keep := true
+			for _, filter := range filters {
+				if filter.pattern.MatchString(name) {
+					keep = filter.keep
+				}
+			}
+
+			if keep {
+				fmt.Printf("%s %s %d\n", name, value, timestamp)
+			}
+		}
+	}
+}
+
+func sleepInterval(interval int) time.Time {
+	now := time.Now().UTC()
+	next := now.Add(time.Duration(interval) * time.Second).
+		Truncate(time.Duration(interval) * time.Second)
+	time.Sleep(next.Sub(now))
+
+	return next
+}
+
+func httpPoll(url string, interval int) {
+	client := http.Client{
+		Timeout: 5 * time.Second,
+	}
+
+	for {
+		now := sleepInterval(interval)
+
+		fmt.Printf("now: %v\n", time.Now().UTC())
+
+		req, _ := http.NewRequest("GET", url, nil)
+		req.Header.Set("User-Agent", userAgent)
+		res, err := client.Get(url)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "http error: %s\n", err)
+			continue
+		}
+
+		if res.StatusCode != 200 {
+			fmt.Fprintf(os.Stderr, "http status: %d\n", res.StatusCode)
+			continue
+		}
+
+		responseHandler(res.Body, now.UnixMilli())
+	}
+}
+
+func filePoll(path string, interval int) {
+	for {
+		now := sleepInterval(interval)
+
+		fmt.Printf("now: %v\n", time.Now().UTC())
+
+		file, err := os.Open(path)
+		if err != nil {
+			log.Print(err)
+			// handle the error and return
+			continue
+		}
+
+		fi, err := file.Stat()
+
+		switch mode := fi.Mode(); {
+		case mode.IsRegular():
+			responseHandler(file, now.UnixMilli())
+		case mode.IsDir():
+			file.Close()
+			files, err := filepath.Glob(path + "/*.prom")
+			if err != nil {
+				log.Fatal(err)
+			}
+			for _, filename := range files {
+				file, err := os.Open(filename)
+				if err != nil {
+					log.Print(err)
+					// handle the error and return
+					continue
+				}
+
+				responseHandler(file, now.UnixMilli())
+			}
+		}
+	}
+}
+
+func pipePoll(command string, interval int) {
+	for {
+		now := sleepInterval(interval)
+
+		fmt.Printf("now sh: %v\n", time.Now().UTC())
+
+		var cmd *exec.Cmd
+		if strings.Index(command, " ") != -1 {
+			cmd = exec.Command("/bin/sh", "-c", command)
+		} else {
+			cmd = exec.Command(command)
+		}
+		stdout, err := cmd.StdoutPipe()
+		if err != nil {
+			log.Print(err)
+		}
+
+		cmd.Start()
+
+		responseHandler(stdout, now.UnixMilli())
+
+		err = cmd.Wait()
+		if exiterr, ok := err.(*exec.ExitError); ok {
+			log.Printf("command exited with status %d\n",
+				exiterr.ExitCode())
+		}
+	}
+}
+
+func main() {
+	if len(os.Args) != 2 {
+		fmt.Fprintf(os.Stderr, "usage: %s configfile\n", os.Args[0])
+		os.Exit(-1)
+	}
+
+	file, err := os.Open(os.Args[1])
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer file.Close()
+
+	interval := 60
+	interval = 1
+
+	scanner := bufio.NewScanner(file)
+	for scanner.Scan() {
+		line := scanner.Text()
+
+		if line == "" || strings.HasPrefix(line, "#") {
+			continue
+		} else if match, _ := regexp.MatchString(`^https?:`, line); match {
+			go httpPoll(line, interval)
+		} else if match, _ := regexp.MatchString(`^file://`, line); match {
+			go filePoll(line[7:], interval)
+		} else if match, _ := regexp.MatchString(`^/`, line); match {
+			go filePoll(line, interval)
+		} else if strings.HasPrefix(line, "-") || strings.HasPrefix(line, "+") {
+			pattern, err := regexp.Compile(strings.TrimSpace(line[1:]))
+			if err != nil {
+				log.Fatal(err)
+			}
+
+			filters = append(filters, Filter{line[0] == '+', pattern})
+		} else if strings.HasPrefix(line, "$") {
+			go pipePoll(strings.TrimSpace(line[1:]), interval)
+		} else if strings.HasPrefix(line, "@") {
+			i, err := strconv.Atoi(strings.TrimSpace(line[1:]))
+			if err != nil {
+				log.Fatal(err)
+			}
+			interval = i
+		} else {
+			log.Fatalf("invalid config line: %s\n", line)
+		}
+	}
+
+	if err := scanner.Err(); err != nil {
+		log.Fatal(err)
+	}
+
+	fmt.Println("hi")
+
+	select {}
+}