Python操作InfluxDB

InfluxDB API Client Libraries

上一篇文章介绍了安装部署InfluxDB和它的一些基本概念，接着就得来处理Nginx access.log，并将处理结果存储在InfluxDB中。
InfluxDB支持多种语言使用其客户端库来进行交互，具体参见官方文档：
https://docs.influxdata.com/influxdb/v1.4/tools/api_client_libraries/

我这里使用Python语言来处理Nginx access.log并将结果存储于InfluxDB中。

下载安装模块influxdb

# pip install influxdb

编写代码

Nginx access.log的一行日志如下：

line ='res.wisedu.com 172.16.6.4 [20/Dec/2017:09:20:17 +0800] "GET /statistics/res?/bh_apis/1.0/module-bhMenu.html&callback=__jp0 HTTP/1.1" 200 0 "-" "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)" 0.000 -'

代码如下：

# -*- coding: utf-8 -*-

import re
from influxdb import InfluxDBClient

def read_log(path): # 生成器generator
    '''一行一行读取日志并返回'''
    with open(path) as f:
        yield from f

def write_influxDB(lst):
    '''写入InfluxDB数据库'''
    client.write_points(lst)

def regular_line(line):
    '''利用正则分析一行日志，存于字典中'''
    o = re.compile(pattern)
    m = o.search(line)
    field_dict = m.groupdict()

    return field_dict

def main():
    '''主函数'''
    for line in read_log(path):
        field_dict = regular_line(line)
        lst = []
        point_dict = {}
        point_dict['measurement'] = 'res_access_log'
        point_dict['fields'] = field_dict
        lst.append(point_dict)

        write_influxDB(lst)

if __name__ == '__main__':
    pattern = '(?P<host>[\w+\.]+\w+) (?P<ip>\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}) \[(?P<time_local>.*)\]'
    pattern += ' "(?P<method>\w+) (?P<url>[^\s]*) (?P<version>[\w\/\d\.]*)" (?P<status>\d+) (?P<length>\d+)'
    pattern += ' "(?P<http_referer>[^\s]*)" "(?P<ua>.*)" (?P<request_time>[\d\.]*) (?P<upstream_response_time>[\d\.]*)'

    path = "logs/res.statistics.log"
    client = InfluxDBClient(host='172.16.7.151', port=8086, username='root', password='wisedu123', database='mydb')

    main()

Chronograf查看表数据