Initial commit

server/elastic/apply_mappings.py

@@ -0,0 +1,47 @@
+#!/usr/bin/env python3
+"""
+Apply elasticsearch mappings to given hosts.
+"""
+import argparse
+import json
+import requests
+from pathlib import Path
+
+HERE = Path(__file__).parent
+
+
+def apply_mappings_to_host(host: str):
+    def _send_mapping(f):
+        with f.open() as json_data:
+            data = json.load(json_data)
+            es_server = host
+            url = f"{es_server}/_template/{f.stem}"
+            requests.delete(url)
+            r = requests.post(
+                url, headers={"Content-Type": "application/json"}, data=json.dumps(data)
+            )
+            return {"mapping": f.stem, "result": r.text}
+
+    p = HERE / "mappings"
+    return [
+        _send_mapping(f) for f in p.iterdir() if f.is_file() and f.suffix == ".json"
+    ]
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(
+        description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter
+    )
+    parser.add_argument("hosts", nargs="+")
+    return parser.parse_args()
+
+
+def main():
+    for host in parse_args().hosts:
+        print(">>>>> Applying mapping to " + host)
+        res = apply_mappings_to_host(host)
+        print(res)
+
+
+if __name__ == "__main__":
+    main()

server/elastic/mappings/events.json

@@ -0,0 +1,27 @@
+{
+  "template": "events-*",
+  "settings": {
+    "number_of_shards": 5
+  },
+  "mappings": {
+    "_default_": {
+      "_source": {
+        "enabled": true
+      },
+      "_routing": {
+        "required": true
+      },
+      "properties": {
+        "@timestamp": { "type": "date" },
+        "task":       { "type": "keyword" },
+        "type":       { "type": "keyword" },
+        "worker":     { "type": "keyword" },
+        "timestamp":  { "type": "date" },
+        "iter":       { "type": "long" },
+        "metric":     { "type": "keyword" },
+        "variant":    { "type": "keyword" },
+        "value":      { "type": "float" }
+      }
+    }
+  }
+}

server/elastic/mappings/events_log.json

@@ -0,0 +1,12 @@
+{
+  "template": "events-log-*",
+  "order" : 1,
+  "mappings": {
+    "_default_": {
+      "properties": {
+        "msg":    { "type":"text", "index": false },
+        "level":  { "type":"keyword" }
+      }
+    }
+  }
+}

server/elastic/mappings/events_plot.json

@@ -0,0 +1,11 @@
+{
+  "template": "events-plot-*",
+  "order" : 1,
+  "mappings": {
+    "_default_": {
+      "properties": {
+        "plot_str": { "type":"text", "index": false }
+      }
+    }
+  }
+}

server/elastic/mappings/events_training_debug_image.json

@@ -0,0 +1,12 @@
+{
+  "template": "events-training_debug_image-*",
+  "order" : 1,
+  "mappings": {
+    "_default_": {
+      "properties": {
+        "key":      { "type": "keyword" },
+        "url":      { "type": "keyword" }
+      }
+    }
+  }
+}

server/elastic/requirements.txt

@@ -0,0 +1 @@
+requests>=2.21.0