http_flow.cpp

#include <cstdio>
#include <pcap.h>
#include <pcre.h>
#include <arpa/inet.h>
#include <getopt.h>
#include <cerrno>
#include <string>
#include <sstream>
#include <list>
#include <map>
#include <sys/types.h>
#include <sys/stat.h>
#include <unistd.h>
#include "util.h"
#include "stream_parser.h"
#include "data_link.h"

#define HTTPFLOW_VERSION "0.0.9"

#define MAXIMUM_SNAPLEN 262144

struct capture_config {
#define IFNAMSIZ    16
    int snaplen;
    std::string output_path;
    char device[IFNAMSIZ];
    std::string file_name;
    std::string filter;
    pcre *url_filter_re;
    pcre_extra *url_filter_extra;
    int datalink_size;
};

std::map<std::string, stream_parser *> http_requests;

struct tcphdr {
    uint16_t th_sport;       /* source port */
    uint16_t th_dport;       /* destination port */
    uint32_t th_seq;         /* sequence number */
    uint32_t th_ack;         /* acknowledgement number */
#define TH_OFF(th)  (((th)->th_offx2 & 0xf0) >> 4)
    uint8_t th_offx2;       /* data offset, rsvd */
/* TCP flags */
#define TH_FIN     0x01
#define TH_SYN     0x02
#define TH_RST     0x04
#define TH_PUSH    0x08
#define TH_ACK     0x10
#define TH_URG     0x20
#define TH_ECNECHO 0x40 /* ECN Echo */
#define TH_CWR     0x80 /* ECN Cwnd Reduced */
    uint8_t th_flags;
    uint16_t th_win;         /* window */
    uint16_t th_sum;         /* checksum */
    uint16_t th_urp;         /* urgent pointer */
};

static bool process_tcp(struct packet_info *packet, const u_char *content, size_t len) {
    if (len < sizeof(struct tcphdr)) {
        std::cerr << "received truncated TCP datagram." << std::endl;
        return false;
    }
    const struct tcphdr *tcp_header = reinterpret_cast<const struct tcphdr *>(content);

    size_t tcp_header_len = TH_OFF(tcp_header) << 2;
    if (len < tcp_header_len) {
        std::cerr << "received truncated TCP datagram." << std::endl;
        return false;
    }

    uint16_t src_port = ntohs(tcp_header->th_sport);
    uint16_t dst_port = ntohs(tcp_header->th_dport);

    char buff[128];
    std::snprintf(buff, 128, "%s:%d", packet->src_addr.c_str(), src_port);
    packet->src_addr.assign(buff);
    std::snprintf(buff, 128, "%s:%d", packet->dst_addr.c_str(), dst_port);
    packet->dst_addr.assign(buff);
    packet->is_syn = tcp_header->th_flags & TH_SYN;
    packet->is_fin = tcp_header->th_flags & TH_FIN;
    packet->is_rst = tcp_header->th_flags & TH_RST;

    content += tcp_header_len;
    packet->body = std::string(reinterpret_cast<const char *>(content), len - tcp_header_len);
    packet->seq = htonl(tcp_header->th_seq);
    packet->ack = htonl(tcp_header->th_ack);
    if (tcp_header->th_flags & (TH_FIN | TH_SYN)) {
        packet->nxtseq = packet->seq + 1;
    } else {
        packet->nxtseq = packet->seq + packet->body.size();
    }
    return true;
}

struct ip {
    uint8_t ip_vhl;     /* header length, version */
#define IP_V(ip)    (((ip)->ip_vhl & 0xf0) >> 4)
#define IP_HL(ip)   ((ip)->ip_vhl & 0x0f)
    uint8_t ip_tos;     /* type of service */
    uint16_t ip_len;     /* total length */
    uint16_t ip_id;      /* identification */
    uint16_t ip_off;     /* fragment offset field */
#define IP_DF 0x4000            /* dont fragment flag */
#define IP_MF 0x2000            /* more fragments flag */
#define IP_OFFMASK 0x1fff       /* mask for fragmenting bits */
    uint8_t ip_ttl;     /* time to live */
    uint8_t ip_p;       /* protocol */
    uint16_t ip_sum;     /* checksum */
    uint32_t ip_src, ip_dst;  /* source and dest address */
};

struct ip6_hdr {
    union {
        struct ip6_hdrctl {
            uint32_t ip6_un1_flow;  /* 20 bits of flow-ID */
            uint16_t ip6_un1_plen;  /* payload length */
            uint8_t ip6_un1_nxt;   /* next header */
            uint8_t ip6_un1_hlim;  /* hop limit */
        } ip6_un1;
        uint8_t ip6_un2_vfc;    /* 4 bits version, top 4 bits class */
    } ip6_ctlun;
    struct in6_addr ip6_src;    /* source address */
    struct in6_addr ip6_dst;    /* destination address */
} UNALIGNED;

#define ip6_plen    ip6_ctlun.ip6_un1.ip6_un1_plen
#define ip6_nxt     ip6_ctlun.ip6_un1.ip6_un1_nxt

static bool process_ipv4(struct packet_info *packet, const u_char *content, size_t len) {
    if (len < sizeof(struct ip)) {
        std::cerr << "received truncated IP datagram." << std::endl;
        return false;
    }
    const struct ip *ip_header = reinterpret_cast<const struct ip *>(content);
    const struct ip6_hdr *ip6_header = NULL;
    if (6 == IP_V(ip_header)) {
        ip6_header = reinterpret_cast<const struct ip6_hdr *>(content);
    }

    size_t ip_payload_len;
    if (ip6_header) {
        if (ip6_header->ip6_nxt != IPPROTO_TCP) {
            return false;
        }
        ip_payload_len = ntohs(ip6_header->ip6_plen);
        size_t ip_header_len = sizeof(struct ip6_hdr);
        if (len < ip_header_len + ip_payload_len) {
            std::cerr << "received truncated IP6 datagram." << std::endl;
            return false;
        }
        content += ip_header_len;

        char addr[INET6_ADDRSTRLEN];
        inet_ntop(AF_INET6, &ip6_header->ip6_src, addr, INET6_ADDRSTRLEN);
        packet->src_addr.assign(addr);
        inet_ntop(AF_INET6, &ip6_header->ip6_dst, addr, INET6_ADDRSTRLEN);
        packet->dst_addr.assign(addr);
    } else {
        if (ip_header->ip_p != IPPROTO_TCP) {
            return false;
        }
        size_t ip_header_len = IP_HL(ip_header) << 2;
        size_t ip_len = ntohs(ip_header->ip_len);
        if (ip_len > len || ip_len < ip_header_len) {
            std::cerr << "received truncated IP4 datagram." << std::endl;
            return false;
        }
        ip_payload_len = ip_len - ip_header_len;
        content += ip_header_len;

        char addr[INET_ADDRSTRLEN];
        inet_ntop(AF_INET, &ip_header->ip_src, addr, INET_ADDRSTRLEN);
        packet->src_addr.assign(addr);
        inet_ntop(AF_INET, &ip_header->ip_dst, addr, INET_ADDRSTRLEN);
        packet->dst_addr.assign(addr);
    }

    return process_tcp(packet, content, ip_payload_len);
}

#define    ETHER_ADDR_LEN      6
#define    ETHERTYPE_IP        0x0800    /* IP protocol */

struct ether_header {
    u_char ether_dhost[ETHER_ADDR_LEN];
    u_char ether_shost[ETHER_ADDR_LEN];
    u_short ether_type;
};

void process_packet(const pcre *url_filter_re, const pcre_extra *url_filter_extra, const std::string &output_path,
                    const u_char *data, size_t len, long ts_usc) {

    struct packet_info packet;
    packet.ts_usc = ts_usc;
    bool ret = process_ipv4(&packet, data, len);
    if (!ret) return;

    std::string join_addr;
    get_join_addr(packet.src_addr, packet.dst_addr, join_addr);
    std::map<std::string, stream_parser *>::iterator iter = http_requests.find(join_addr);

    if (iter == http_requests.end()) {
        if (!packet.body.empty() || (packet.is_syn && packet.ack == 0)) {
            stream_parser *parser = new stream_parser(url_filter_re, url_filter_extra, output_path);
            if (parser->parse(packet, HTTP_REQUEST)) {
                parser->set_addr(packet.src_addr, packet.dst_addr);
                http_requests.insert(std::make_pair(join_addr, parser));
            } else {
                delete parser;
            }
        }
    } else {
        stream_parser *parser = iter->second;
        enum http_parser_type type = parser->is_request_address(packet.src_addr) ? HTTP_REQUEST : HTTP_RESPONSE;
        if (!parser->parse(packet, type)) {
            delete parser;
            http_requests.erase(iter);
        }
    }
}

void pcap_callback(u_char *arg, const struct pcap_pkthdr *header, const u_char *content) {

    // Data: |       Mac         |        Ip          |           TCP                  |
    // Len : |   ETHER_HDR_LEN   |   ip_header->ip_hl << 2   | tcp_header->th_off << 2 + sizeof body |

    capture_config *conf = reinterpret_cast<capture_config *>(arg);

    // skip datalink
    content += conf->datalink_size;
    size_t len = header->caplen - conf->datalink_size;
    long ts = header->ts.tv_sec * 1000000 + header->ts.tv_usec;

    return process_packet(conf->url_filter_re, conf->url_filter_extra, conf->output_path, content, len, ts);
}

static const struct option longopts[] = {
        {"help",        no_argument,       NULL, 'h'},
        {"interface",   required_argument, NULL, 'i'},
        {"url-filter",  required_argument, NULL, 'u'},
        {"pcap-file",   required_argument, NULL, 'r'},
        {"output-path", required_argument, NULL, 'w'},
        {NULL, 0,                          NULL, 0}
};

#define SHORTOPTS "hi:u:r:w:"

int print_usage() {
    std::cerr << "libpcap version " << pcap_lib_version() << "\n"
              << "httpflow version " HTTPFLOW_VERSION "\n"
              << "\n"
              << "Usage: httpflow [-i interface | -r pcap-file] [-u url-filter] [-w output-path] [expression]"
              << "\n"
              << "\n"
              << "  -i interface      Listen on interface, This is same as tcpdump 'interface'\n"
              << "  -r pcap-file      Read packets from file (which was created by tcpdump with the -w option)\n"
              << "                    Standard input is used if file is '-'\n"
              << "  -u url-filter     Matches which urls will be dumped\n"
              << "  -w output-path    Write the http request and response to a specific directory\n"
              << "\n"
              << "  expression        Selects which packets will be dumped, The format is the same as tcpdump's 'expression' argument\n"
              << "                    If filter expression is given, only packets for which expression is 'true' will be dumped\n"
              << "                    For the expression syntax, see pcap-filter(7)" << "\n\n"
              << "  For more information, see https://github.com/six-ddc/httpflow" << "\n\n";
    exit(0);
}

extern char *optarg;            /* getopt(3) external variables */
extern int optind, opterr, optopt;

capture_config *default_config() {
    capture_config *conf = new capture_config;

    conf->snaplen = MAXIMUM_SNAPLEN;
    conf->device[0] = 0;
    conf->filter = "tcp";
    conf->url_filter_re = NULL;
    conf->url_filter_extra = NULL;

    return conf;
}

static std::string copy_argv(char **argv) {
    char **p;
    size_t len = 0;
    char *src;
    std::string buf;
    std::string::iterator dst;

    p = argv;
    if (*p == NULL)
        return buf;

    while (*p)
        len += strlen(*p++) + 1;

    buf.assign(len - 1, 0);
    if (buf.empty()) {
        return buf;
    }

    p = argv;
    dst = buf.begin();
    while ((src = *p++) != NULL) {
        if (src != *argv) {
            *(dst - 1) = ' ';
        }
        while ((*dst++ = *src++) != '\0');
    }

    return buf;
}

int init_capture_config(int argc, char **argv, capture_config *conf, char *errbuf) {

    // pcap_if_t *devices = NULL, *iter = NULL;
    const char *default_device = NULL;
    int cnt, op, i;
    std::string url_regex;

    while ((op = getopt_long(argc, argv, SHORTOPTS, longopts, NULL)) != -1) {
        switch (op) {
            case 'i':
                std::strncpy(conf->device, optarg, sizeof(conf->device));
                break;
            case 'u':
                url_regex.assign(optarg);
                const char *err;
                int erroffset;
                conf->url_filter_re = pcre_compile(url_regex.c_str(), 0, &err, &erroffset, NULL);
                if (!conf->url_filter_re) {
                    std::cerr << "invalid regular expression at offset " << erroffset << ": " << err << std::endl;
                    exit(1);
                }
                conf->url_filter_extra = pcre_study(conf->url_filter_re, 0, &err);
                break;
            case 'r':
                conf->file_name = optarg;
                break;
            case 'h':
                print_usage();
                break;
            case 'w':
                conf->output_path = optarg;
                break;
            default:
                exit(1);
                break;
        }
    }

    if (optind < argc) {
        conf->filter = copy_argv(&argv[optind]);
    }

    if (conf->device[0] == 0) {
        default_device = pcap_lookupdev(errbuf);
        if (default_device) {
            std::strncpy(conf->device, default_device, sizeof(conf->device));
        }
    }

    if (!conf->output_path.empty()) {
        int mk_ret = mkdir(conf->output_path.c_str(), S_IRWXU);
        if (mk_ret != 0 && errno != EEXIST) {
            std::cerr << "mkdir [" << conf->output_path << "] failed. ret=" << mk_ret << std::endl;
            exit(1);
        }
    }

    if (conf->file_name.empty()) {
        std::cerr << "interface: " << conf->device << std::endl;
    } else {
        if (conf->file_name == "-") {
            std::cerr << "pcap-file: [stdin]" << std::endl;
        } else {
            std::cerr << "pcap-file: " << conf->file_name << std::endl;
        }
    }
    if (!conf->output_path.empty()) {
        std::cerr << "output_path: " << conf->output_path << std::endl;
    }
    std::cerr << "filter: " << conf->filter << std::endl;
    if (!url_regex.empty()) {
        std::cerr << "url_filter: " << url_regex << std::endl;
    }

    return 0;
}

int main(int argc, char **argv) {
    is_atty = isatty(fileno(stdout));

    char errbuf[PCAP_ERRBUF_SIZE];
    pcap_t *handle = NULL;
    bpf_u_int32 net = 0, mask = 0;
    struct bpf_program fcode;
    int datalink_id;
    std::string datalink_str;

    capture_config *cap_conf = default_config();
    if (-1 == init_capture_config(argc, argv, cap_conf, errbuf)) {
        return 1;
    }

    if (!cap_conf->file_name.empty()) {
        handle = pcap_open_offline(cap_conf->file_name.c_str(), errbuf);
        if (!handle) {
            std::cerr << "pcap_open_offline(): " << errbuf << std::endl;
            return 1;
        }
    } else {
        pcap_lookupnet(cap_conf->device, &net, &mask, errbuf);

        handle = pcap_open_live(cap_conf->device, cap_conf->snaplen, 0, 1000, errbuf);
        if (!handle) {
            std::cerr << "pcap_open_live(): " << errbuf << std::endl;
            return 1;
        }

        pcap_datalink(handle);
    }

    if (-1 == pcap_compile(handle, &fcode, cap_conf->filter.c_str(), 0, mask)) {
        std::cerr << "pcap_compile(): " << pcap_geterr(handle) << std::endl;
        pcap_close(handle);
        return 1;
    }

    if (-1 == pcap_setfilter(handle, &fcode)) {
        std::cerr << "pcap_setfilter(): " << pcap_geterr(handle) << std::endl;
        pcap_close(handle);
        return 1;
    }

    pcap_freecode(&fcode);

    datalink_id = pcap_datalink(handle);
    datalink_str = datalink2str(datalink_id);
    cap_conf->datalink_size = datalink2off(datalink_id);
    std::cerr << "datalink: " << datalink_id << "(" << datalink_str << ") header size: " << cap_conf->datalink_size
              << std::endl;

    if (-1 == pcap_loop(handle, -1, pcap_callback, reinterpret_cast<u_char *>(cap_conf))) {
        std::cerr << "pcap_loop(): " << pcap_geterr(handle) << std::endl;
        pcap_close(handle);
        return 1;
    }

    delete cap_conf;

    pcap_close(handle);
    return 0;
}