https://github.com/AliyunContainerService/log-pilot

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
apiVersion: v1
kind: ConfigMap
metadata:
labels:
cattle.io/creator: norman
name: filebeat
namespace: kube-system
data:
filebeat.tpl: |-
{{range .configList}}
- type: log
enabled: true
paths:
- {{ .HostDir }}/{{ .File }}
multiline.pattern: '(^\d{4}-\d{1,2}-\d{1,2}\s\d{1,2}:\d{1,2}:\d{1,2}|^([A-a]lert|ALERT|[T|t]race|TRACE|[D|d]ebug|DEBUG|[N|n]otice|NOTICE|[I|i]nfo|INFO|[W|w]arn?(?:ing)?|WARN?(?:ING)?|[E|e]rr?(?:or)?|ERR?(?:OR)?|[C|c]rit?(?:ical)?|CRIT?(?:ICAL)?|[F|f]atal|FATAL|[S|s]evere|SEVERE|EMERG(?:ENCY)?|[Ee]merg(?:ency)?))'
multiline.negate: true
multiline.match: after
scan_frequency: 10s
fields_under_root: true
{{if .Stdout}}
docker-json: true
{{end}}
{{if eq .Format "json"}}
json.keys_under_root: true
{{end}}
fields:
{{range $key, $value := .Tags}}
{{ $key }}: {{ $value }}
{{end}}
{{range $key, $value := $.container}}
{{ $key }}: {{ $value }}
{{end}}
tail_files: false
close_inactive: 2h
close_eof: false
close_removed: true
clean_removed: true
close_renamed: false
{{end}}
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
labels:
app: log-pilot
name: log-pilot
namespace: kube-system
spec:
selector:
matchLabels:
app: log-pilot
template:
metadata:
labels:
app: log-pilot
spec:
containers:
- env:
- name: KAFKA_BROKERS
value: ***.***.***.***:9092,***.***.***.***:9092,***.***.***.***:9092
- name: LOGGING_OUTPUT
value: kafka
- name: NODE_NAME
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: spec.nodeName
image: registry.cn-hangzhou.aliyuncs.com/acs/log-pilot:0.9.7-filebeat
imagePullPolicy: IfNotPresent
livenessProbe:
exec:
command:
- /pilot/healthz
failureThreshold: 3
initialDelaySeconds: 10
periodSeconds: 10
successThreshold: 1
timeoutSeconds: 2
name: log-pilot
resources:
limits:
memory: 500Mi
requests:
cpu: 200m
memory: 200Mi
securityContext:
capabilities:
add:
- SYS_ADMIN
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: File
volumeMounts:
- mountPath: /pilot/filebeat.tpl
name: filebeat
subPath: filebeat.tpl
- mountPath: /var/run/docker.sock
name: sock
- mountPath: /host
name: root
readOnly: true
- mountPath: /var/lib/filebeat
name: varlib
- mountPath: /var/log/filebeat
name: varlog
- mountPath: /etc/localtime
name: localtime
readOnly: true
tolerations:
- effect: NoSchedule
key: node-role.kubernetes.io/master
volumes:
- configMap:
defaultMode: 420
name: filebeat
name: filebeat
- hostPath:
path: /var/run/docker.sock
type: ""
name: sock
- hostPath:
path: /
type: ""
name: root
- hostPath:
path: /var/lib/filebeat
type: DirectoryOrCreate
name: varlib
- hostPath:
path: /var/log/filebeat
type: DirectoryOrCreate
name: varlog
- hostPath:
path: /etc/localtime
type: ""
name: localtime
updateStrategy:
rollingUpdate:
maxUnavailable: 1
type: RollingUpdate

阿里云kafka ca-cert

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
-----BEGIN CERTIFICATE-----
MIIDPDCCAqWgAwIBAgIJAMRsb0DLM1fsMA0GCSqGSIb3DQEBBQUAMHIxCzAJBgNV
BAYTAkNOMQswCQYDVQQIEwJIWjELMAkGA1UEBxMCSFoxCzAJBgNVBAoTAkFCMRAw
DgYDVQQDEwdLYWZrYUNBMSowKAYJKoZIhvcNAQkBFht6aGVuZG9uZ2xpdS5semRA
YWxpYmFiYS5jb20wIBcNMTcwMzA5MTI1MDUyWhgPMjEwMTAyMTcxMjUwNTJaMHIx
CzAJBgNVBAYTAkNOMQswCQYDVQQIEwJIWjELMAkGA1UEBxMCSFoxCzAJBgNVBAoT
AkFCMRAwDgYDVQQDEwdLYWZrYUNBMSowKAYJKoZIhvcNAQkBFht6aGVuZG9uZ2xp
dS5semRAYWxpYmFiYS5jb20wgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBALZV
bbIO1ULQQN853BTBgRfPiRJaAOWf38u8GC0TNp/E9qtI88A+79ywAP17k5WYJ7XS
wXMOJ3h1qkQT2TYJVetZ6E69CUJq4BsOvNlNRvmnW6eFymh5QZsEz2MTooxJjVjC
JQPlI2XRDjIrTVYEQWUDxj2JhB8VVPEed+6u4KQVAgMBAAGjgdcwgdQwHQYDVR0O
BBYEFHFlOoiqQxXanVi2GUoDiKDD33ujMIGkBgNVHSMEgZwwgZmAFHFlOoiqQxXa
nVi2GUoDiKDD33ujoXakdDByMQswCQYDVQQGEwJDTjELMAkGA1UECBMCSFoxCzAJ
BgNVBAcTAkhaMQswCQYDVQQKEwJBQjEQMA4GA1UEAxMHS2Fma2FDQTEqMCgGCSqG
SIb3DQEJARYbemhlbmRvbmdsaXUubHpkQGFsaWJhYmEuY29tggkAxGxvQMszV+ww
DAYDVR0TBAUwAwEB/zANBgkqhkiG9w0BAQUFAAOBgQBTSz04p0AJXKl30sHw+UM/
/k1jGFJzI5p0Z6l2JzKQYPP3PfE/biE8/rmiGYEenNqWNy1ZSniEHwa8L/Ux98ci
4H0ZSpUrMo2+6bfuNW9X35CFPp5vYYJqftilJBKIJX3C3J1ruOuBR28UxE42xx4K
pQ70wChNi914c4B+SxkGUg==
-----END CERTIFICATE-----

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
output.kafka:
## Kafka 实例的接入点
hosts: ["***.***.***.***:9093", "***.***.***.***:9093", "***.***.***.***:9093"]
## 控制台获取
username: "kafka"
password: "***"

# 指定topic的名称
topic: 'project_mysql_logs'
# 指定阿里云kafka版本
version: 0.10.2
required_acks: 1
max_message_bytes: 1000000

# 配置数字证书, 请把Demo里面的ca-cert放到某个位置,然后把路径填在这里
ssl.certificate_authorities: ["ca-cert"]
# 取消证书有效性校验
ssl.verification_mode: none