diff --git a/img/20210408121947.png b/img/20210408121947.png new file mode 100644 index 0000000..a33fd3a Binary files /dev/null and b/img/20210408121947.png differ diff --git a/img/20210408142609.png b/img/20210408142609.png new file mode 100644 index 0000000..731e5fc Binary files /dev/null and b/img/20210408142609.png differ diff --git a/img/20210408142805.png b/img/20210408142805.png new file mode 100644 index 0000000..23e9c28 Binary files /dev/null and b/img/20210408142805.png differ diff --git a/img/20210409165112.png b/img/20210409165112.png new file mode 100644 index 0000000..a23d865 Binary files /dev/null and b/img/20210409165112.png differ diff --git a/img/20210410191428.png b/img/20210410191428.png new file mode 100644 index 0000000..0eb7dcd Binary files /dev/null and b/img/20210410191428.png differ diff --git a/img/20210412105544.png b/img/20210412105544.png new file mode 100644 index 0000000..ba26c4b Binary files /dev/null and b/img/20210412105544.png differ diff --git a/img/20210412111812.png b/img/20210412111812.png new file mode 100644 index 0000000..7113837 Binary files /dev/null and b/img/20210412111812.png differ diff --git a/img/20210412115250.png b/img/20210412115250.png new file mode 100644 index 0000000..281868f Binary files /dev/null and b/img/20210412115250.png differ diff --git a/img/20210412115743.png b/img/20210412115743.png new file mode 100644 index 0000000..604731f Binary files /dev/null and b/img/20210412115743.png differ diff --git a/img/20210412134121.png b/img/20210412134121.png new file mode 100644 index 0000000..fc3e37c Binary files /dev/null and b/img/20210412134121.png differ diff --git a/img/20210412134505.png b/img/20210412134505.png new file mode 100644 index 0000000..dde01a1 Binary files /dev/null and b/img/20210412134505.png differ diff --git a/img/20210412165734.png b/img/20210412165734.png new file mode 100644 index 0000000..62c54a6 Binary files /dev/null and b/img/20210412165734.png differ diff --git a/img/20210412170701.png b/img/20210412170701.png new file mode 100644 index 0000000..fa3b5d0 Binary files /dev/null and b/img/20210412170701.png differ diff --git a/img/20210412173041.png b/img/20210412173041.png new file mode 100644 index 0000000..11229f8 Binary files /dev/null and b/img/20210412173041.png differ diff --git a/img/20210416144601.png b/img/20210416144601.png new file mode 100644 index 0000000..dee6011 Binary files /dev/null and b/img/20210416144601.png differ diff --git a/img/20210416145923.png b/img/20210416145923.png new file mode 100644 index 0000000..bfb3c95 Binary files /dev/null and b/img/20210416145923.png differ diff --git a/img/20210416151130.png b/img/20210416151130.png new file mode 100644 index 0000000..d73906f Binary files /dev/null and b/img/20210416151130.png differ diff --git a/img/20210416153841.png b/img/20210416153841.png new file mode 100644 index 0000000..a91c71e Binary files /dev/null and b/img/20210416153841.png differ diff --git a/img/20210416155448.png b/img/20210416155448.png new file mode 100644 index 0000000..464df43 Binary files /dev/null and b/img/20210416155448.png differ diff --git a/img/20210416163447.png b/img/20210416163447.png new file mode 100644 index 0000000..2ac8e8b Binary files /dev/null and b/img/20210416163447.png differ diff --git a/img/20210416163824.png b/img/20210416163824.png new file mode 100644 index 0000000..527a2d3 Binary files /dev/null and b/img/20210416163824.png differ diff --git a/img/20210416165340.png b/img/20210416165340.png new file mode 100644 index 0000000..fedd0ec Binary files /dev/null and b/img/20210416165340.png differ diff --git a/img/20210416171427.png b/img/20210416171427.png new file mode 100644 index 0000000..6c320f4 Binary files /dev/null and b/img/20210416171427.png differ diff --git a/img/20210416171541.png b/img/20210416171541.png new file mode 100644 index 0000000..5f08493 Binary files /dev/null and b/img/20210416171541.png differ diff --git a/img/20210416173045.png b/img/20210416173045.png new file mode 100644 index 0000000..2f74530 Binary files /dev/null and b/img/20210416173045.png differ diff --git a/img/20210802151326.png b/img/20210802151326.png new file mode 100644 index 0000000..88e53df Binary files /dev/null and b/img/20210802151326.png differ diff --git a/img/20210922095549.png b/img/20210922095549.png new file mode 100644 index 0000000..26d99ab Binary files /dev/null and b/img/20210922095549.png differ diff --git a/img/20210922101132.png b/img/20210922101132.png new file mode 100644 index 0000000..6f1074d Binary files /dev/null and b/img/20210922101132.png differ diff --git a/img/20210922143659.png b/img/20210922143659.png new file mode 100644 index 0000000..ce6fb4b Binary files /dev/null and b/img/20210922143659.png differ diff --git a/img/20210922144109.png b/img/20210922144109.png new file mode 100644 index 0000000..07b8466 Binary files /dev/null and b/img/20210922144109.png differ diff --git a/img/20210922144924.png b/img/20210922144924.png new file mode 100644 index 0000000..7c9411b Binary files /dev/null and b/img/20210922144924.png differ diff --git a/img/20211003163143.png b/img/20211003163143.png new file mode 100644 index 0000000..0a412d2 Binary files /dev/null and b/img/20211003163143.png differ diff --git a/img/20211004141744.png b/img/20211004141744.png new file mode 100644 index 0000000..c11fd39 Binary files /dev/null and b/img/20211004141744.png differ diff --git a/img/20211004181409.png b/img/20211004181409.png new file mode 100644 index 0000000..5e294a4 Binary files /dev/null and b/img/20211004181409.png differ diff --git a/img/20211004182439.png b/img/20211004182439.png new file mode 100644 index 0000000..7f6d838 Binary files /dev/null and b/img/20211004182439.png differ diff --git a/img/20211005120542.png b/img/20211005120542.png new file mode 100644 index 0000000..59194b1 Binary files /dev/null and b/img/20211005120542.png differ diff --git a/img/20211005122945.png b/img/20211005122945.png new file mode 100644 index 0000000..2b29d76 Binary files /dev/null and b/img/20211005122945.png differ diff --git a/img/20211005142120.png b/img/20211005142120.png new file mode 100644 index 0000000..b338116 Binary files /dev/null and b/img/20211005142120.png differ diff --git a/img/20211006160513.png b/img/20211006160513.png new file mode 100644 index 0000000..513eb5c Binary files /dev/null and b/img/20211006160513.png differ diff --git a/img/20211006162136.png b/img/20211006162136.png new file mode 100644 index 0000000..c8d2a62 Binary files /dev/null and b/img/20211006162136.png differ diff --git a/img/prometheus-metric-type-counter.jpeg b/img/prometheus-metric-type-counter.jpeg new file mode 100644 index 0000000..09bb655 Binary files /dev/null and b/img/prometheus-metric-type-counter.jpeg differ diff --git a/img/prometheus-metric-type-gauge.jpeg b/img/prometheus-metric-type-gauge.jpeg new file mode 100644 index 0000000..e6df644 Binary files /dev/null and b/img/prometheus-metric-type-gauge.jpeg differ diff --git a/img/prometheus-metric-type-histogram.jpeg b/img/prometheus-metric-type-histogram.jpeg new file mode 100644 index 0000000..b228038 Binary files /dev/null and b/img/prometheus-metric-type-histogram.jpeg differ diff --git a/img/prometheus-metric-type-summary.jpeg b/img/prometheus-metric-type-summary.jpeg new file mode 100644 index 0000000..6cf481c Binary files /dev/null and b/img/prometheus-metric-type-summary.jpeg differ diff --git a/prometheus/01 基本概念.md b/prometheus/01 基本概念.md index 255b2b2..6016e98 100644 --- a/prometheus/01 基本概念.md +++ b/prometheus/01 基本概念.md @@ -441,4 +441,5 @@ rate(prometheus_tsdb_head_samples_appended_total[1m]) ![prometheus_tsdb_head_samples_appended_total rate](../img/20210408113024.png) -我们可以在 `Table` 和 `Graph` 视图下面切换查看表达式查询的结果。 \ No newline at end of file +我们可以在 `Table` 和 `Graph` 视图下面切换查看表达式查询的结果。 + diff --git a/prometheus/02 PromQL.md b/prometheus/02 PromQL.md new file mode 100644 index 0000000..738ed82 --- /dev/null +++ b/prometheus/02 PromQL.md @@ -0,0 +1,383 @@ +# PromQL 介绍 + +**PromQL** 是 Prometheus 监控系统内置的一种查询语言，PromQL 允许你以灵活的方式选择、聚合等其他方式转换和计算时间序列数据，该语言仅用于读取数据。可以说 PromQL 是我们学习 Prometheus 最困难也是最重要的部分，本章节我们将介绍 PromQL 的基础知识、理论基础，然后会深入了解更加高级的查询模式。 + +## 目标 + +通过对本章节 PromQL 的学习你将能够有效地构建、分享和理解 PromQL 查询，可以帮助我们从容应对报警规则、仪表盘可视化等需求，还能够避免一些在使用 PromQL 表达式的时候遇到的一些陷进。 + +## 执行 + +前面基础章节我们介绍了 Prometheus 整体的架构： + +![Prometheus 架构中的 PromQL](../img/20210408121947.png) + +当 Prometheus 从系统和服务收集指标数据时，它会把数据存储在内置的时序数据库（TSDB）中，要对收集到的数据进行任何处理，我们都可以使用 PromQL 从 TSDB 中读取数据，同时可以对所选的数据执行过滤、聚合以及其他转换操作。 + +PromQL 的执行可以通过两种方式来触发： + +- 在 Prometheus 服务器中，记录规则和警报规则会定期运行，并执行查询操作来计算规则结果（例如触发报警）。该执行在 Prometheus 服务内部进行，并在配置规则时自动发生。 +- 外部用户和 UI 界面可以使用 Prometheus 服务提供的 [HTTP API](https://prometheus.io/docs/prometheus/latest/querying/api/) 来执行 PromQL 查询。这就是仪表盘软件（例如 [Grafana](https://grafana.com/grafana/)、[PromLens](https://promlens.com/) 以及 Prometheus 内置 Web UI）访问 PromQL 的方式。 + +## 场景 + +PromQL 可以用于许多监控场景，下面简单介绍几个相关案例。 + +**临时查询** + +我们可以用 PromQL 来对收集的数据进行实时查询，这有助于我们去调试和诊断遇到的一些问题，我们一般也是直接使用内置的表达式查询界面来执行这类查询： + +![PromQL 查询](../img/20210408142609.png) + +**仪表盘** + +同样我们也可以基于 PromQL 查询来创建可视化的图形、表格等面板，当然一般我们都会使用 Grafana： + +![grafana dashboard](../img/20210408142805.png) + +Grafana 原生支持 Prometheus 作为数据源，并内置支持了 PromQL 表达式的查询。 + +**报警** + +Prometheus 可以直接使用基于 PromQL 对收集的数据进行的查询结果来生成报警，一个完整的报警规则如下所示： + +```yaml +groups: + - name: demo-service-alerts + rules: + - alert: Many5xxErrors + expr: | + ( + sum by(path, instance, job) ( + rate(demo_api_request_duration_seconds_count{status=~"5..",job="demo"}[1m]) + ) + / + sum by(path, instance, job) ( + rate(demo_api_request_duration_seconds_count{job="demo"}[1m]) + ) * 100 > 0.5 + ) + for: 30s + labels: + severity: critical + annotations: + title: '{{$labels.instance}} high 5xx rate on {{$labels.path}}' + description: 'The 5xx error rate for path {{$labels.path}} on {{$labels.instance}} is {{$value}}%.' +``` + + + +除了构成报警规则核心的 PromQL 表达式（上面 YAML 文件中的 `expr` 属性），报警规则还包含其他的一些元数据字段，后面在具体讲解报警的章节中会详细和大家讲解。 + +然后，Prometheus 可以通过一个名为 `Alertmanager` 的组件来发送报警通知，可以配置一些接收器来接收这些报警，比如用钉钉来接收报警： + +![钉钉](../img/20211003163143.png) + +**自动化** + +此外我们还可以构建自动化流程，针对 PromQL 执行的查询结果来做出决策，比如 Kubernetes 中基于自定义指标的 HPA。 + + + +# 数据模型 + + + +在开始学习 PromQL 的知识之前，我们先重新来熟悉下 Prometheus 的数据模型 + +![Prometheus 数据模型](../img/20210409165112.png) + +## 时间序列 + +Prometheus 会将所有采集到的样本数据以时间序列的方式保存在**内存数据库**中，并且定时保存到硬盘上。时间序列是按照时间戳和值的序列顺序存放的，我们称之为向量(vector)，每条时间序列通过指标名称(metrics name)和一组标签集(labelset)命名。如下所示，可以将时间序列理解为一个以时间为 X 轴的数字矩阵： + +```text + ^ + │ . . . . . . . . . . . . . . . . . . . node_cpu_seconds_total{cpu="cpu0",mode="idle"} + │ . . . . . . . . . . . . . . . . . . . node_cpu_seconds_total{cpu="cpu0",mode="system"} + │ . . . . . . . . . . . . . . . . . . node_load1{} + │ . . . . . . . . . . . . . . . . . . + v + <------------------ 时间 ----------------> +``` + + + +在时间序列中的每一个点称为一个样本（sample），样本由以下三部分组成： + +- 指标(metric)：指标名和描述当前样本特征的标签集合 +- 时间戳(timestamp)：一个精确到毫秒的时间戳 +- 样本值(value)： 一个 float64 的浮点型数据表示当前样本的值 + +如下所示： + +```text +<--------------- metric ---------------------><-timestamp -><-value-> +http_request_total{status="200", method="GET"}@1434417560938 => 94355 +http_request_total{status="200", method="GET"}@1434417561287 => 94334 + +http_request_total{status="404", method="GET"}@1434417560938 => 38473 +http_request_total{status="404", method="GET"}@1434417561287 => 38544 + +http_request_total{status="200", method="POST"}@1434417560938 => 4748 +http_request_total{status="200", method="POST"}@1434417561287 => 4785 +``` + + + +在形式上，所有的指标都通过如下格式表示： + +```text +{