libtrace_int.h

Go to the documentation of this file.

/*
 *
 * Copyright (c) 2007-2016 The University of Waikato, Hamilton, New Zealand.
 * All rights reserved.
 *
 * This file is part of libtrace.
 *
 * This code has been developed by the University of Waikato WAND
 * research group. For further information please see http://www.wand.net.nz/
 *
 * libtrace is free software; you can redistribute it and/or modify
 * it under the terms of the GNU Lesser General Public License as published by
 * the Free Software Foundation; either version 3 of the License, or
 * (at your option) any later version.
 *
 * libtrace is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 *
 *
 */
#ifndef LIBTRACE_INT_H
#define LIBTRACE_INT_H

#ifdef __cplusplus
extern "C" {
#endif

#include "config.h"
#include "common.h"
#include "libtrace_parallel.h"
#include "wandio.h"
#include "lt_bswap.h"

#ifdef _MSC_VER
// warning: deprecated function
#pragma warning(disable:4996)
// warning: benign redefinitions of types
#pragma warning(disable:4142)
#endif

#ifdef HAVE_INTTYPES_H
# include <inttypes.h>
#else
# include "lt_inttypes.h"
#endif

#ifdef HAVE_STDDEF_H
# include <stddef.h>
#else
#ifndef WIN32
# error "Can't find stddev.h -- do you define ptrdiff_t elsewhere?"
#endif
#endif


#include "rt_protocol.h"
        
/* Prefer net/bpf.h over pcap-bpf.h for format_bpf.c on MacOS */
#ifdef HAVE_NET_BPF_H
#    include <net/bpf.h>
#    define HAVE_BPF 1
#else
#ifdef HAVE_PCAP_BPF_H
#  include <pcap-bpf.h>
#  define HAVE_BPF 1
#endif
#endif

#ifdef HAVE_PCAP_H
#  include <pcap.h>
#  ifdef HAVE_PCAP_INT_H
#    include <pcap-int.h>
#  endif
#endif 

#ifdef HAVE_ZLIB_H
#  include <zlib.h>
#endif

#if !HAVE_DECL_STRNDUP
char *strndup(const char *s, size_t size);
#endif

#if !HAVE_DECL_STRNCASECMP
# ifndef HAVE__STRNICMP

int strncasecmp(const char *str1, const char *str2, size_t n);
# else
# define strncasecmp _strnicmp
# endif
#endif

#if !HAVE_DECL_SNPRINTF
# ifndef HAVE_SPRINTF_S

int snprintf(char *str, size_t size, const char *format, ...);
# else
# define snprintf sprintf_s
# endif 
#endif

#include "daglegacy.h"
        
#ifdef HAVE_DAG_API
#  include "dagnew.h"
#  include "dagapi.h"
#       if DAG_VERSION == 24
#               include <erftypes.h>
#       else
#               include <daginf.h>
#       endif
#  include "erftypes.h"
#else
#  include "dagformat.h"
#endif

#ifdef HAVE_LLVM
#include "bpf-jit/bpf-jit.h"
#endif

#include "data-struct/ring_buffer.h"
#include "data-struct/object_cache.h"
#include "data-struct/vector.h"
#include "data-struct/message_queue.h"
#include "data-struct/deque.h"
#include "data-struct/linked_list.h"
#include "data-struct/sliding_window.h"
#include "data-struct/buckets.h"
#include "pthread_spinlock.h"

//#define RP_BUFSIZE 65536U

#define LIBTRACE_MAX_REPLAY_SPEEDUP 1000

struct libtrace_event_status_t {
        libtrace_packet_t *packet;

        /* The walltime when we processed the first packet from the trace */
        double first_now;

        /* The tracetime of the first packet in the trace */
        double first_ts;

        int psize;
        bool waiting;
};

enum thread_types {
        THREAD_EMPTY,
        THREAD_HASHER,
        THREAD_PERPKT,
        THREAD_REPORTER,
        THREAD_KEEPALIVE
};

enum thread_states {
        THREAD_RUNNING,
        THREAD_FINISHING,
        THREAD_FINISHED,
        THREAD_PAUSED,
        THREAD_STATE_MAX
};

enum hash_owner {
        HASH_OWNED_LIBTRACE,
        HASH_OWNED_EXTERNAL,
};

struct libtrace_thread_t {
        uint64_t accepted_packets; // The number of packets accepted only used if pread
        uint64_t filtered_packets;
        // is retreving packets
        // Set to true once the first packet has been stored
        bool recorded_first;
        // For thread safety reason we actually must store this here
        int64_t tracetime_offset_usec;
        void* user_data; // TLS for the user to use
        void* format_data; // TLS for the format to use
        libtrace_message_queue_t messages; // Message handling
        libtrace_ringbuffer_t rbuffer; // Input
        libtrace_t * trace;
        void* ret;
        enum thread_types type;
        enum thread_states state;
        pthread_t tid;
        int perpkt_num; // A number from 0-X that represents this perpkt threads number
                                // in the table, intended to quickly identify this thread
                                // -1 represents NA (such as the case this is not a perpkt thread)
} ALIGN_STRUCT(CACHE_LINE_SIZE);

struct first_packets {
        pthread_spinlock_t lock;
        size_t count; // If == perpkt_thread_count threads we have all
        size_t first; // Valid if count != 0
        struct {
                libtrace_packet_t * packet;
                struct timeval tv;
        } * packets;
};

#define TRACE_STATES \
        X(STATE_NEW) \
        X(STATE_RUNNING) \
        X(STATE_PAUSING) \
        X(STATE_PAUSED) \
        X(STATE_FINISHED) \
        X(STATE_FINISHING) \
        X(STATE_DESTROYED) \
        X(STATE_JOINED) \
        X(STATE_ERROR)

#define X(a) a,
enum trace_state {
        TRACE_STATES
};
#undef X

#define X(a) case a: return #a;
static inline char *get_trace_state_name(enum trace_state ts){
        switch(ts) {
                TRACE_STATES
                default:
                        return "UNKNOWN";
        }
}
#undef X

#define READ_EOF 0
#define READ_ERROR -1
#define READ_MESSAGE -2
// Used for inband tick message
#define READ_TICK -3

struct user_configuration {
        size_t cache_size;
        size_t thread_cache_size;
        bool fixed_count;
        size_t burst_size;
        size_t tick_interval;
        size_t tick_count;
        size_t perpkt_threads;
        size_t hasher_queue_size;
        bool hasher_polling;
        bool reporter_polling;
        size_t reporter_thold;
        bool debug_state;
};
#define ZERO_USER_CONFIG(config) memset(&config, 0, sizeof(struct user_configuration));

struct callback_set {

        fn_cb_starting message_starting;
        fn_cb_dataless message_stopping;
        fn_cb_dataless message_resuming;
        fn_cb_dataless message_pausing;
        fn_cb_packet message_packet;
        fn_cb_result message_result;
        fn_cb_first_packet message_first_packet;
        fn_cb_tick message_tick_count;
        fn_cb_tick message_tick_interval;
        fn_cb_usermessage message_user;
};

struct libtrace_t {
        struct libtrace_format_t *format; 
        struct libtrace_event_status_t event;
        void *format_data;              
        struct libtrace_filter_t *filter; 
        size_t snaplen;                 
        int replayspeedup;
        uint64_t accepted_packets;
        uint64_t filtered_packets;
        uint64_t sequence_number;
        libtrace_packet_t *last_packet;
        char *uridata;
        io_t *io;
        libtrace_err_t err;
        bool started;
        int startcount;
        pthread_mutex_t libtrace_lock;
        pthread_mutex_t read_packet_lock;
        enum trace_state state;
        pthread_cond_t perpkt_cond;
        int perpkt_thread_states[THREAD_STATE_MAX]; 

        bool perpkt_queue_full;
        void* global_blob;
        libtrace_ocache_t packet_freelist;
        enum hasher_types hasher_type;
        fn_hasher hasher;
        void *hasher_data;
        enum hash_owner hasher_owner;
        int (*pread)(libtrace_t *, libtrace_thread_t *, libtrace_packet_t **, size_t);

        libtrace_thread_t hasher_thread;
        libtrace_thread_t reporter_thread;
        libtrace_thread_t keepalive_thread;
        int perpkt_thread_count;
        libtrace_thread_t * perpkt_threads; // All our perpkt threads
        // Used to keep track of the first packet seen on each thread
        struct first_packets first_packets;
        int tracetime;

        /*
         * Caches statistic counters in the case that our trace is
         * paused or stopped before this counter is taken
         */
        libtrace_stat_t *stats;
        struct user_configuration config;
        libtrace_combine_t combiner;
        
        /* Set of callbacks to be executed by per packet threads in response
         * to various messages. */
        struct callback_set *perpkt_cbs;
        /* Set of callbacks to be executed by the reporter thread in response
         * to various messages. */
        struct callback_set *reporter_cbs;
};

#define LIBTRACE_STAT_MAGIC 0x41

void trace_fin_packet(libtrace_packet_t *packet);
void libtrace_zero_thread(libtrace_thread_t * t);
void store_first_packet(libtrace_t *libtrace, libtrace_packet_t *packet, libtrace_thread_t *t);
libtrace_thread_t * get_thread_table(libtrace_t *libtrace);


void send_message(libtrace_t *trace, libtrace_thread_t *target,
                const enum libtrace_messages type,
                libtrace_generic_t data, libtrace_thread_t *sender);

struct libtrace_out_t {
        struct libtrace_format_t *format;
        void *format_data;              
        char *uridata;                  
        libtrace_err_t err;
        bool started;
};

void trace_set_err(libtrace_t *trace, int errcode,const char *msg,...) 

                                                                PRINTF(3,4);
void trace_set_err_out(libtrace_out_t *trace, int errcode, const char *msg,...)
                                                                PRINTF(3,4);

void trace_clear_cache(libtrace_packet_t *packet);


#ifndef PF_RULESET_NAME_SIZE
#define PF_RULESET_NAME_SIZE 16
#endif

#ifndef IFNAMSIZ
#define IFNAMSIZ 16
#endif


typedef struct libtrace_pflog_header_t {
        uint8_t    length;      
        sa_family_t   af;
        uint8_t    action;
        uint8_t    reason;
        char       ifname[IFNAMSIZ];
        char       ruleset[PF_RULESET_NAME_SIZE];
        uint32_t   rulenr;
        uint32_t   subrulenr;
        uint8_t    dir;
        uint8_t    pad[3];
} PACKED libtrace_pflog_header_t;

/* All functions should return -1, or NULL on failure */
struct libtrace_format_t {
        const char *name;
        const char *version;
        enum base_format_t type;


        int (*probe_filename)(const char *fname);
        
        int (*probe_magic)(io_t *io);

        int (*init_input)(libtrace_t *libtrace);
        
        int (*config_input)(libtrace_t *libtrace,trace_option_t option,void *value);
        int (*start_input)(libtrace_t *libtrace);

        int (*pause_input)(libtrace_t *libtrace);

        int (*init_output)(libtrace_out_t *libtrace);
        
        int (*config_output)(libtrace_out_t *libtrace, trace_option_output_t option, void *value);

        int (*start_output)(libtrace_out_t *libtrace);

        int (*fin_input)(libtrace_t *libtrace);

        int (*fin_output)(libtrace_out_t *libtrace);

        int (*read_packet)(libtrace_t *libtrace, libtrace_packet_t *packet);
        
        int (*prepare_packet)(libtrace_t *libtrace, libtrace_packet_t *packet,
                        void *buffer, libtrace_rt_types_t rt_type, 
                        uint32_t flags);
        
        void (*fin_packet)(libtrace_packet_t *packet);

        int (*write_packet)(libtrace_out_t *libtrace, libtrace_packet_t *packet);

        int (*flush_output)(libtrace_out_t *libtrace);

        libtrace_linktype_t (*get_link_type)(const libtrace_packet_t *packet);

        libtrace_direction_t (*get_direction)(const libtrace_packet_t *packet);
        
        libtrace_direction_t (*set_direction)(libtrace_packet_t *packet, libtrace_direction_t direction);
        
        uint64_t (*get_erf_timestamp)(const libtrace_packet_t *packet);

        struct timeval (*get_timeval)(const libtrace_packet_t *packet);
        
        struct timespec (*get_timespec)(const libtrace_packet_t *packet);
        
        double (*get_seconds)(const libtrace_packet_t *packet);
        
        int (*seek_erf)(libtrace_t *trace, uint64_t timestamp);
        int (*seek_timeval)(libtrace_t *trace, struct timeval tv);
        
        int (*seek_seconds)(libtrace_t *trace, double seconds);
        
        int (*get_capture_length)(const libtrace_packet_t *packet);

        int (*get_wire_length)(const libtrace_packet_t *packet);
        
        int (*get_framing_length)(const libtrace_packet_t *packet);

        size_t (*set_capture_length)(struct libtrace_packet_t *packet,size_t size);
        uint64_t (*get_received_packets)(libtrace_t *trace);

        uint64_t (*get_filtered_packets)(libtrace_t *trace);
        
        uint64_t (*get_dropped_packets)(libtrace_t *trace);

        void (*get_statistics)(libtrace_t *trace, libtrace_stat_t *stat);
        
        int (*get_fd)(const libtrace_t *trace);
        
        struct libtrace_eventobj_t (*trace_event)(libtrace_t *trace, libtrace_packet_t *packet);        

        void (*help)(void);
        
        struct libtrace_format_t *next;

        struct libtrace_info_t info;

        int (*pstart_input)(libtrace_t *trace);
        
        int (*pread_packets)(libtrace_t *trace, libtrace_thread_t *t, libtrace_packet_t **packets, size_t nb_packets);
        
        int (*ppause_input)(libtrace_t *trace);
        
        int (*pfin_input)(libtrace_t *trace);

        int (*pregister_thread)(libtrace_t *libtrace, libtrace_thread_t *t, bool reader);

        void (*punregister_thread)(libtrace_t *libtrace, libtrace_thread_t *t);

        void (*get_thread_statistics)(libtrace_t *libtrace,
                                      libtrace_thread_t *t,
                                      libtrace_stat_t *stat);
};

#define NON_PARALLEL(live) \
        {live, 1},              /* trace info */ \
        NULL,                   /* pstart_input */ \
        NULL,                   /* pread_packet */ \
        NULL,                   /* ppause_input */ \
        NULL,                   /* pfin_input */ \
        NULL,                   /* pregister_thread */ \
        NULL,                   /* punregister_thread */ \
        NULL,                   /* get_thread_statistics */

//extern struct libtrace_format_t *form;

extern volatile int libtrace_halt;

static inline int is_halted(libtrace_t *trace) {
        if (!(libtrace_halt || trace->state == STATE_PAUSING)) {
                return -1;
        } else if (libtrace_halt) {
                return READ_EOF;
        } else {
                return READ_MESSAGE;
        }
}

void register_format(struct libtrace_format_t *format);

uint64_t tv_to_usec(const struct timeval *tv);

libtrace_linktype_t pcap_linktype_to_libtrace(libtrace_dlt_t linktype);

libtrace_rt_types_t pcap_linktype_to_rt(libtrace_dlt_t linktype);

libtrace_rt_types_t pcapng_linktype_to_rt(libtrace_dlt_t linktype);

libtrace_dlt_t libtrace_to_pcap_linktype(libtrace_linktype_t type);

libtrace_dlt_t libtrace_to_pcap_dlt(libtrace_linktype_t type);

libtrace_dlt_t rt_to_pcap_linktype(libtrace_rt_types_t rt_type);

libtrace_rt_types_t bpf_linktype_to_rt(libtrace_dlt_t linktype);

libtrace_linktype_t erf_type_to_libtrace(uint8_t erf);

uint8_t libtrace_to_erf_type(libtrace_linktype_t linktype);

libtrace_linktype_t arphrd_type_to_libtrace(unsigned int arphrd);

unsigned int libtrace_to_arphrd_type(libtrace_linktype_t type);

void promote_packet(libtrace_packet_t *packet);

bool demote_packet(libtrace_packet_t *packet);

void *trace_get_payload_from_linux_sll(const void *link,
                uint16_t *arphrd_type, 
                uint16_t *next_header, 
                uint32_t *remaining);

DLLEXPORT void *trace_get_payload_from_atm(void *link, uint8_t *type, 
                uint32_t *remaining);


#ifdef HAVE_BPF
/* A type encapsulating a bpf filter
 * This type covers the compiled bpf filter, as well as the original filter
 * string
 *
 */

struct libtrace_filter_t {
        struct bpf_program filter;      
        char * filterstring;            
        int flag;                       
        struct bpf_jit_t *jitfilter;
};
#else

struct libtrace_filter_t {};
#endif

typedef struct libtrace_pcapfile_pkt_hdr_t {
        uint32_t ts_sec;        /* Seconds portion of the timestamp */
        uint32_t ts_usec;       /* Microseconds portion of the timestamp */
        uint32_t caplen;        /* Capture length of the packet */
        uint32_t wirelen;       /* The wire length of the packet */
} libtrace_pcapfile_pkt_hdr_t;

#ifdef HAVE_DAG

void dag_constructor(void);
#endif

void erf_constructor(void);
void tsh_constructor(void);
void legacy_constructor(void);
void linuxnative_constructor(void);
void linuxring_constructor(void);
void pcap_constructor(void);
void pcapfile_constructor(void);
void pcapng_constructor(void);
void rt_constructor(void);
void duck_constructor(void);
void atmhdr_constructor(void);
void ndag_constructor(void);
void etsilive_constructor(void);
#ifdef HAVE_BPF

void bpf_constructor(void);
#endif
#if HAVE_DPDK

void dpdk_constructor(void);

void dpdkndag_constructor(void);

#endif

bool trace_get_wireless_flags(void *link, libtrace_linktype_t linktype, uint8_t *flags);
#define TRACE_RADIOTAP_F_FCS 0x10
        
#ifdef __cplusplus
}
#endif

#endif /* LIBTRACE_INT_H */