diff options
Diffstat (limited to 'gprofng/common/hwcdrv.c')
| -rw-r--r-- | gprofng/common/hwcdrv.c | 1454 |
1 files changed, 1454 insertions, 0 deletions
diff --git a/gprofng/common/hwcdrv.c b/gprofng/common/hwcdrv.c new file mode 100644 index 0000000..caab983 --- /dev/null +++ b/gprofng/common/hwcdrv.c @@ -0,0 +1,1454 @@ +/* Copyright (C) 2021 Free Software Foundation, Inc. + Contributed by Oracle. + + This file is part of GNU Binutils. + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, 51 Franklin Street - Fifth Floor, Boston, + MA 02110-1301, USA. */ + +#include <errno.h> +#include <unistd.h> +#include <fcntl.h> +#include <sys/mman.h> +#include <sys/ioctl.h> +#include <sys/syscall.h> +#include <linux/perf_event.h> + +#include "hwcdrv.h" + +/*---------------------------------------------------------------------------*/ +/* macros */ +#define IS_GLOBAL /* Mark global symbols */ + +#include "cpuid.c" /* ftns for identifying a chip */ + +static hdrv_pcbe_api_t hdrv_pcbe_core_api; +static hdrv_pcbe_api_t hdrv_pcbe_opteron_api; +static hdrv_pcbe_api_t *hdrv_pcbe_drivers[] = { + &hdrv_pcbe_core_api, + &hdrv_pcbe_opteron_api, + NULL +}; +#include "opteron_pcbe.c" /* CPU-specific code */ +#include "core_pcbe.c" /* CPU-specific code */ + +extern hwcdrv_api_t hwcdrv_pcl_api; +IS_GLOBAL hwcdrv_api_t *hwcdrv_drivers[] = { + &hwcdrv_pcl_api, + NULL +}; + +/*---------------------------------------------------------------------------*/ + +/* utils for drivers */ +IS_GLOBAL int +hwcdrv_assign_all_regnos (Hwcentry* entries[], unsigned numctrs) +{ + unsigned int pmc_assigned[MAX_PICS]; + unsigned idx; + for (int ii = 0; ii < MAX_PICS; ii++) + pmc_assigned[ii] = 0; + + /* assign the HWCs that we already know about */ + for (idx = 0; idx < numctrs; idx++) + { + regno_t regno = entries[idx]->reg_num; + if (regno == REGNO_ANY) + { + /* check to see if list of possible registers only contains one entry */ + regno = REG_LIST_SINGLE_VALID_ENTRY (entries[idx]->reg_list); + } + if (regno != REGNO_ANY) + { + if (regno < 0 || regno >= MAX_PICS || !regno_is_valid (entries[idx], regno)) + { + logerr (GTXT ("For counter #%d, register %d is out of range\n"), idx + 1, regno); /*!*/ + return HWCFUNCS_ERROR_HWCARGS; + } + TprintfT (DBG_LT2, "hwcfuncs_assign_regnos(): preselected: idx=%d, regno=%d\n", idx, regno); + entries[idx]->reg_num = regno; /* assigning back to entries */ + pmc_assigned[regno] = 1; + } + } + + /* assign HWCs that are currently REGNO_ANY */ + for (idx = 0; idx < numctrs; idx++) + { + if (entries[idx]->reg_num == REGNO_ANY) + { + int assigned = 0; + regno_t *reg_list = entries[idx]->reg_list; + for (; reg_list && *reg_list != REGNO_ANY; reg_list++) + { + regno_t regno = *reg_list; + if (regno < 0 || regno >= MAX_PICS) + { + logerr (GTXT ("For counter #%d, register %d is out of range\n"), idx + 1, regno); /*!*/ + return HWCFUNCS_ERROR_HWCARGS; + } + if (pmc_assigned[regno] == 0) + { + TprintfT (DBG_LT2, "hwcfuncs_assign_regnos(): assigned: idx=%d, regno=%d\n", idx, regno); + entries[idx]->reg_num = regno; /* assigning back to entries */ + pmc_assigned[regno] = 1; + assigned = 1; + break; + } + } + if (!assigned) + { + logerr (GTXT ("Counter '%s' could not be bound to a register\n"), + entries[idx]->name ? entries[idx]->name : "<NULL>"); + return HWCFUNCS_ERROR_HWCARGS; + } + } + } + return 0; +} + +IS_GLOBAL int +hwcdrv_lookup_cpuver (const char * cpcN_cciname) +{ + libcpc2_cpu_lookup_t *plookup; + static libcpc2_cpu_lookup_t cpu_table[] = { + LIBCPC2_CPU_LOOKUP_LIST + }; + if (cpcN_cciname == NULL) + return CPUVER_UNDEFINED; + + /* search table for name */ + for (plookup = cpu_table; plookup->cpc2_cciname; plookup++) + { + int n = strlen (plookup->cpc2_cciname); + if (!strncmp (plookup->cpc2_cciname, cpcN_cciname, n)) + return plookup->cpc2_cpuver; + } + /* unknown, but does have a descriptive string */ + TprintfT (DBG_LT0, "hwcfuncs: CPC2: WARNING: Id of processor '%s' " + "could not be determined\n", + cpcN_cciname); + return CPUVER_GENERIC; +} + +/*---------------------------------------------------------------------------*/ +/* utils to generate x86 register definitions on Linux */ + +/* + * This code is structured as though we're going to initialize the + * HWC by writing the Intel MSR register directly. That is, we + * assume the lowest 16 bits of the event number will have the event + * and that higher bits will set attributes. + * + * While SPARC is different, we can nonetheless use basically the + * same "x86"-named functions: + * + * - The event code will still be 16 bits. It will still + * be in the lowest 16 bits of the event number. Though + * perf_event_code() on SPARC will expect those bits to + * shifted, hwcdrv_pcl.c can easily perform that shift. + * + * - On SPARC we support only two attributes, "user" and "system", + * which hwcdrv_pcl.c already converts to the "exclude_user" + * and "exclude_kernel" fields expected by perf_event_open(). + * "user" and "system" are stored in event bits 16 and 17. + * For M8, a 4-bit mask of supported PICs is stored in bits [23:20]. + */ + +IS_GLOBAL hwcdrv_get_eventnum_fn_t *hwcdrv_get_x86_eventnum = 0; + +static const attr_info_t perfctr_sparc_attrs[] = { + {NTXT ("user"), 0, 0x01, 16}, //usr + {NTXT ("system"), 0, 0x01, 17}, //os + {NULL, 0, 0x00, 0}, +}; +static const attr_info_t perfctr_x64_attrs[] = {/* ok for Core2 & later */ + {NTXT ("umask"), 0, 0xff, 8}, + {NTXT ("user"), 0, 0x01, 16}, //usr + //{NTXT("nouser"), 1, 0x01, 16}, //usr (inverted) + {NTXT ("system"), 0, 0x01, 17}, //os + {NTXT ("edge"), 0, 0x01, 18}, + {NTXT ("pc"), 0, 0x01, 19}, + {NTXT ("inv"), 0, 0x01, 23}, + {NTXT ("cmask"), 0, 0xff, 24}, + {NULL, 0, 0x00, 0}, +}; +const attr_info_t *perfctr_attrs_table = perfctr_x64_attrs; + +static const eventsel_t perfctr_evntsel_enable_bits = (0x01 << 16) | /* usr */ + // (0xff << 0) | /* event*/ + // (0xff << 8) | /* umask */ + // (0x01 << 17) | /* os */ + // (0x01 << 18) | /* edge */ + // (0x01 << 19) | /* pc */ + (0x01 << 20) | /* int */ + // (0x01 << 21) | /* reserved */ + (0x01 << 22) | /* enable */ + // (0x01 << 23) | /* inv */ + // (0xff << 24) | /* cmask */ + 0; + +static int +myperfctr_get_x86_eventnum (const char *eventname, uint_t pmc, + eventsel_t *eventsel, eventsel_t *valid_umask, + uint_t *pmc_sel) +{ + if (hwcdrv_get_x86_eventnum && + !hwcdrv_get_x86_eventnum (eventname, pmc, eventsel, valid_umask, pmc_sel)) + return 0; + + /* check for numerically-specified counters */ + char * endptr; + uint64_t num = strtoull (eventname, &endptr, 0); + if (*eventname && !*endptr) + { + *eventsel = EXTENDED_EVNUM_2_EVSEL (num); + *valid_umask = 0xff; /* allow any umask (unused for SPARC?) */ + *pmc_sel = pmc; + return 0; + } + + /* name does not specify a numeric value */ + *eventsel = (eventsel_t) - 1; + *valid_umask = 0x0; + *pmc_sel = pmc; + return -1; +} + +static int +mask_shift_set (eventsel_t *presult, eventsel_t invalue, + eventsel_t mask, eventsel_t shift) +{ + if (invalue & ~mask) + return -1; /* invalue attempts to set bits outside of mask */ + *presult &= ~(mask << shift); /* clear all the mask bits */ + *presult |= (invalue << shift); /* set bits according to invalue */ + return 0; +} + +static int +set_x86_attr_bits (eventsel_t *result_mask, eventsel_t evnt_valid_umask, + hwcfuncs_attr_t attrs[], int nattrs, const char*nameOnly) +{ + eventsel_t evntsel = *result_mask; + for (int ii = 0; ii < (int) nattrs; ii++) + { + const char *attrname = attrs[ii].ca_name; + eventsel_t attrval = (eventsel_t) attrs[ii].ca_val; + const char *tmpname; + int attr_found = 0; + for (int jj = 0; (tmpname = perfctr_attrs_table[jj].attrname); jj++) + { + if (strcmp (attrname, tmpname) == 0) + { + if (strcmp (attrname, "umask") == 0) + { + if (attrval & ~evnt_valid_umask) + { + logerr (GTXT ("for `%s', allowable umask bits are: 0x%llx\n"), + nameOnly, (long long) evnt_valid_umask); + return -1; + } + } + if (mask_shift_set (&evntsel, + perfctr_attrs_table[jj].is_inverted ? (attrval^1) : attrval, + perfctr_attrs_table[jj].mask, + perfctr_attrs_table[jj].shift)) + { + logerr (GTXT ("`%s' attribute `%s' could not be set to 0x%llx\n"), + nameOnly, attrname, (long long) attrval); + return -1; + } + TprintfT (DBG_LT2, "hwcfuncs: Counter %s, attribute %s set to 0x%llx\n", + nameOnly, attrname, (long long) attrval); + attr_found = 1; + break; + } + } + if (!attr_found) + { + logerr (GTXT ("attribute `%s' is invalid\n"), attrname); + return -1; + } + } + *result_mask = evntsel; + return 0; +} + +IS_GLOBAL int +hwcfuncs_get_x86_eventsel (unsigned int regno, const char *int_name, + eventsel_t *return_event, uint_t *return_pmc_sel) +{ + hwcfuncs_attr_t attrs[HWCFUNCS_MAX_ATTRS + 1]; + unsigned nattrs = 0; + char *nameOnly = NULL; + eventsel_t evntsel = 0; // event number + eventsel_t evnt_valid_umask = 0; + uint_t pmc_sel = 0; + int rc = -1; + *return_event = 0; + *return_pmc_sel = 0; + void *attr_mem = hwcfuncs_parse_attrs (int_name, attrs, HWCFUNCS_MAX_ATTRS, + &nattrs, NULL); + if (!attr_mem) + { + logerr (GTXT ("out of memory, could not parse attributes\n")); + return -1; + } + hwcfuncs_parse_ctr (int_name, NULL, &nameOnly, NULL, NULL, NULL); + if (regno == REGNO_ANY) + { + logerr (GTXT ("reg# could not be determined for `%s'\n"), nameOnly); + goto attr_wrapup; + } + + /* look up evntsel */ + if (myperfctr_get_x86_eventnum (nameOnly, regno, + &evntsel, &evnt_valid_umask, &pmc_sel)) + { + logerr (GTXT ("counter `%s' is not valid\n"), nameOnly); + goto attr_wrapup; + } + TprintfT (DBG_LT1, "hwcfuncs: event=0x%llx pmc=0x%x '%s' nattrs = %u\n", + (long long) evntsel, pmc_sel, nameOnly, nattrs); + + /* determine event attributes */ + eventsel_t evnt_attrs = perfctr_evntsel_enable_bits; + if (set_x86_attr_bits (&evnt_attrs, evnt_valid_umask, attrs, nattrs, nameOnly)) + goto attr_wrapup; + if (evntsel & evnt_attrs) + TprintfT (DBG_LT0, "hwcfuncs: ERROR - evntsel & enable bits overlap: 0x%llx 0x%llx 0x%llx\n", + (long long) evntsel, (long long) evnt_attrs, + (long long) (evntsel & evnt_attrs)); + *return_event = evntsel | evnt_attrs; + *return_pmc_sel = pmc_sel; + rc = 0; + +attr_wrapup: + free (attr_mem); + free (nameOnly); + return rc; +} + +#ifdef __x86_64__ +#define syscall_instr "syscall" +#define syscall_clobber "rcx", "r11", "memory" +#endif +#ifdef __i386__ +#define syscall_instr "int $0x80" +#define syscall_clobber "memory" +#endif + +static inline int +perf_event_open (struct perf_event_attr *hw_event_uptr, pid_t pid, + int cpu, int group_fd, unsigned long flags) +{ + /* It seems that perf_event_open() sometimes fails spuriously, + * even while an immediate retry succeeds. + * So, let's try a few retries if the call fails just to be sure. + */ + int rc; + for (int retry = 0; retry < 5; retry++) + { + rc = syscall (__NR_perf_event_open, hw_event_uptr, pid, cpu, group_fd, flags); + if (rc != -1) + return rc; + } + return rc; +} + +/*---------------------------------------------------------------------------*/ +/* macros & fwd prototypes */ + +#define HWCDRV_API static /* Mark functions used by hwcdrv API */ + +HWCDRV_API int hwcdrv_start (void); +HWCDRV_API int hwcdrv_free_counters (); + +static pid_t +hwcdrv_gettid (void) +{ +#ifndef LIBCOLLECTOR_SRC + return syscall (__NR_gettid); +#elif defined(intel) + pid_t r; + __asm__ __volatile__(syscall_instr + : "=a" (r) : "0" (__NR_gettid) + : syscall_clobber); + return r; +#else + return syscall (__NR_gettid); // FIXUP_XXX_SPARC_LINUX // write gettid in asm +#endif +} + +/*---------------------------------------------------------------------------*/ +/* types */ + +#define NPAGES_PER_BUF 1 // number of pages to be used for perf_event samples +// must be a power of 2 + +/*---------------------------------------------------------------------------*/ + +/* typedefs */ + +typedef struct +{ // event (hwc) definition + unsigned int reg_num; // PMC assignment, potentially for detecting conflicts + eventsel_t eventsel; // raw event bits (Intel/AMD) + uint64_t counter_preload; // number of HWC events before signal + struct perf_event_attr hw; // perf_event definition + hrtime_t min_time; // minimum time we're targeting between events + char *name; +} perf_event_def_t; + +typedef struct +{ // runtime state of perf_event buffer + void *buf; // pointer to mmapped buffer + size_t pagesz; // size of pages +} buffer_state_t; + +typedef struct +{ // runtime state of counter values + uint64_t prev_ena_ts; // previous perf_event "enabled" time + uint64_t prev_run_ts; // previous perf_event "running" time + uint64_t prev_value; // previous HWC value +} counter_value_state_t; + +typedef struct +{ // per-counter information + perf_event_def_t *ev_def; // global HWC definition for one counter + int fd; // perf_event fd + buffer_state_t buf_state; // perf_event buffer's state + counter_value_state_t value_state; // counter state + int needs_restart; // workaround for dbx failure to preserve si_fd + uint64_t last_overflow_period; + hrtime_t last_overflow_time; +} counter_state_t; + +typedef struct +{ // per-thread context + counter_state_t *ctr_list; + int signal_fd; // fd that caused the most recent signal + pthread_t tid; // for debugging signal delivery problems +} hdrv_pcl_ctx_t; + +/*---------------------------------------------------------------------------*/ + +/* static variables */ +static struct +{ + int library_ok; + int internal_open_called; + hwcfuncs_tsd_get_fn_t find_vpc_ctx; + unsigned hwcdef_cnt; /* number of *active* hardware counters */ + hwcdrv_get_events_fn_t *get_events; +} hdrv_pcl_state; + +static hwcdrv_about_t hdrv_pcl_about = {.cpcN_cpuver = CPUVER_UNDEFINED}; +static perf_event_def_t global_perf_event_def[MAX_PICS]; + +#define COUNTERS_ENABLED() (hdrv_pcl_state.hwcdef_cnt) + + +/* perf_event buffer formatting and handling */ +static void +reset_buf (buffer_state_t *bufstate) +{ + TprintfT (0, "hwcdrv: ERROR: perf_event reset_buf() called!\n"); + struct perf_event_mmap_page *metadata = bufstate->buf; + if (metadata) + metadata->data_tail = metadata->data_head; +} + +static int +skip_buf (buffer_state_t *bufstate, size_t sz) +{ + TprintfT (DBG_LT1, "hwcdrv: WARNING: perf_event skip_buf called!\n"); + struct perf_event_mmap_page *metadata = bufstate->buf; + if (metadata == NULL) + return -1; + size_t pgsz = bufstate->pagesz; + size_t bufsz = NPAGES_PER_BUF*pgsz; + uint64_t d_tail = metadata->data_tail; + uint64_t d_head = metadata->data_head; + + // validate request size + if (sz > d_head - d_tail || sz >= bufsz) + { + reset_buf (bufstate); + return -1; + } + metadata->data_tail = d_tail + sz; // advance tail + return 0; +} + +static int +read_buf (buffer_state_t *bufstate, void *buf, size_t sz) +{ + struct perf_event_mmap_page *metadata = bufstate->buf; + if (metadata == NULL) + return -1; + size_t pgsz = bufstate->pagesz; + size_t bufsz = NPAGES_PER_BUF*pgsz; + uint64_t d_tail = metadata->data_tail; + uint64_t d_head = metadata->data_head; + + // validate request size + if (sz > d_head - d_tail || sz >= bufsz) + { + reset_buf (bufstate); + return -1; + } + char *buf_base = ((char *) metadata) + pgsz; // start of data buffer + uint64_t start_pos = d_tail & (bufsz - 1); // char offset into data buffer + size_t nbytes = sz; + if (start_pos + sz > bufsz) + { + // will wrap past end of buffer + nbytes = bufsz - start_pos; + memcpy (buf, buf_base + start_pos, nbytes); + start_pos = 0; // wrap to start + buf = (void *) (((char *) buf) + nbytes); + nbytes = sz - nbytes; + } + memcpy (buf, buf_base + start_pos, nbytes); + metadata->data_tail += sz; + return 0; +} + +static int +read_u64 (buffer_state_t *bufstate, uint64_t *value) +{ + return read_buf (bufstate, value, sizeof (uint64_t)); +} + +static int +read_sample (counter_state_t *ctr_state, int msgsz, uint64_t *rvalue, + uint64_t *rlost) +{ + // returns count of bytes read + buffer_state_t *bufstate = &ctr_state->buf_state; + counter_value_state_t *cntstate = &ctr_state->value_state; + int readsz = 0; + + // PERF_SAMPLE_IP + uint64_t ipc = 0; + int rc = read_u64 (bufstate, &ipc); + if (rc) + return -1; + readsz += sizeof (uint64_t); + + // PERF_SAMPLE_READ: value + uint64_t value = 0; + rc = read_u64 (bufstate, &value); + if (rc) + return -2; + readsz += sizeof (uint64_t); + + /* Bug 20806896 + * Old Linux kernels (e.g. 2.6.32) on certain systems return enabled and + * running times in the sample data that correspond to the metadata times + * metadata->time_enabled + * metadata->time_running + * from the PREVIOUS (not current) sample. Probably just ignore this bug + * since it's on old kernels and we only use the enabled and running times + * to construct loss_estimate. + */ + // PERF_SAMPLE_READ: PERF_FORMAT_ENABLED + uint64_t enabled_time = 0; + rc = read_u64 (bufstate, &enabled_time); + if (rc) + return -3; + readsz += sizeof (uint64_t); + + // PERF_SAMPLE_READ: PERF_FORMAT_RUNNING + uint64_t running_time = 0; + rc = read_u64 (bufstate, &running_time); + if (rc) + return -4; + readsz += sizeof (uint64_t); + + uint64_t value_delta = value - cntstate->prev_value; + uint64_t enabled_delta = enabled_time - cntstate->prev_ena_ts; + uint64_t running_delta = running_time - cntstate->prev_run_ts; + cntstate->prev_value = value; + cntstate->prev_ena_ts = enabled_time; + cntstate->prev_run_ts = running_time; + + // 24830461 need workaround for Linux anomalous HWC skid overrun + int set_error_flag = 0; + if (value_delta > 2 * ctr_state->last_overflow_period + 2000 /* HWC_SKID_TOLERANCE */) + set_error_flag = 1; + + uint64_t loss_estimate = 0; // estimate loss of events caused by multiplexing + if (running_delta == enabled_delta) + { + // counter was running 100% of time, no multiplexing + } + else if (running_delta == 0) + loss_estimate = 1; // token amount to aid in debugging perfctr oddities + else if ((running_delta > enabled_delta) || (enabled_delta & 0x1000000000000000ll)) + { + // running should be smaller than enabled, can't estimate + /* + * 21418391 HWC can have a negative count + * + * We've also seen enabled not only be smaller than running + * but in fact go negative. Guard against this. + */ + loss_estimate = 2; // token amount to aid in debugging perfctr oddities + } + else + { + // counter was running less than 100% of time + // Example: ena=7772268 run=6775669 raw_value=316004 scaled_value=362483 loss_est=46479 + uint64_t scaled_delta = (double) value_delta * enabled_delta / running_delta; + value_delta = scaled_delta; +#if 0 + // We should perhaps warn the user that multiplexing is going on, + // but hwcdrv_pcl.c doesn't know about the collector_interface, SP_JCMD_COMMENT, or COL_COMMENT_* values. + // For now we simply don't report. + // Perhaps we should address the issue not here but in the caller collector_sigemt_handler(), + // but at that level "lost" has a meaning that's considerably broader than just multiplexing. + collector_interface->writeLog ("<event kind=\"%s\" id=\"%d\">%s %d -> %d</event>\n", + SP_JCMD_COMMENT, COL_COMMENT_HWCADJ, global_perf_event_def[idx].name, + ctr_list[idx].last_overflow_period, new_period); +#endif + } + TprintfT ((loss_estimate || set_error_flag) ? DBG_LT1 : DBG_LT3, + "hwcdrv: '%s' ipc=0x%llx ena=%llu run=%llu " + "value_delta=%lld(0x%llx) loss_est=%llu %s error_flag='%s'\n", + ctr_state->ev_def->name, (long long) ipc, + (long long) enabled_delta, (long long) running_delta, + (long long) value_delta, (long long) value_delta, + (unsigned long long) loss_estimate, + loss_estimate ? ", WARNING - SCALED" : "", + set_error_flag ? ", ERRORFLAG" : ""); + if (set_error_flag == 1) + value_delta |= (1ULL << 63) /* HWCVAL_ERR_FLAG */; + *rvalue = value_delta; + *rlost = loss_estimate; + if (readsz != msgsz) + { + TprintfT (0, "hwcdrv: ERROR: perf_event sample not fully parsed\n"); + return -5; + } + return 0; +} + +static void +dump_perf_event_attr (struct perf_event_attr *at) +{ + TprintfT (DBG_LT2, "dump_perf_event_attr: size=%d type=%d sample_period=%lld\n" + " config=0x%llx config1=0x%llx config2=0x%llx wakeup_events=%lld __reserved_1=%lld\n", + (int) at->size, (int) at->type, (unsigned long long) at->sample_period, + (unsigned long long) at->config, (unsigned long long) at->config1, + (unsigned long long) at->config2, (unsigned long long) at->wakeup_events, + (unsigned long long) at->__reserved_1); +#define DUMP_F(fld) if (at->fld) TprintfT(DBG_LT2, " %-10s : %lld\n", #fld, (long long) at->fld) + DUMP_F (disabled); + DUMP_F (inherit); + DUMP_F (pinned); + DUMP_F (exclusive); + DUMP_F (exclude_user); + DUMP_F (exclude_kernel); + DUMP_F (exclude_hv); + DUMP_F (exclude_idle); + // DUMP_F(xmmap); + DUMP_F (comm); + DUMP_F (freq); + DUMP_F (inherit_stat); + DUMP_F (enable_on_exec); + DUMP_F (task); + DUMP_F (watermark); +} + +static void +init_perf_event (struct perf_event_attr *hw, uint64_t event, uint64_t period) +{ + memset (hw, 0, sizeof (struct perf_event_attr)); + hw->size = sizeof (struct perf_event_attr); // fwd/bwd compat + +#if defined(__i386__) || defined(__x86_64) + //note: Nehalem/Westmere OFFCORE_RESPONSE in upper 32 bits + hw->config = event; + hw->type = PERF_TYPE_RAW; // hw/sw/trace/raw... +#elif defined(__aarch64__) + hw->type = (event >> 24) & 7; + hw->config = event & 0xff; +#elif defined(sparc) + //SPARC needs to be shifted up 16 bits + hw->config = (event & 0xFFFF) << 16; // uint64_t event + uint64_t regs = (event >> 20) & 0xf; // see sparc_pcbe.c + hw->config |= regs << 4; // for M8, supported PICs need to be placed at bits [7:4] + hw->type = PERF_TYPE_RAW; // hw/sw/trace/raw... +#endif + + hw->sample_period = period; + hw->sample_type = PERF_SAMPLE_IP | PERF_SAMPLE_READ | + // PERF_SAMPLE_TID | + // PERF_SAMPLE_TIME | // possibly interesting + // PERF_SAMPLE_ADDR | + PERF_SAMPLE_READ | // HWC value + // PERF_SAMPLE_CALLCHAIN | // interesting + // PERF_SAMPLE_ID | + // PERF_SAMPLE_CPU | // possibly interesting + // PERF_SAMPLE_PERIOD | + // PERF_SAMPLE_STREAM_ID | + // PERF_SAMPLE_RAW | + 0; + hw->read_format = + PERF_FORMAT_TOTAL_TIME_ENABLED | // detect when hwc not scheduled + PERF_FORMAT_TOTAL_TIME_RUNNING | // detect when hwc not scheduled + // PERF_FORMAT_ID | + // PERF_FORMAT_GROUP | + 0; + hw->disabled = 1; /* off by default */ + + // Note: the following override config.priv bits! + hw->exclude_user = (event & (1 << 16)) == 0; /* don't count user */ + hw->exclude_kernel = (event & (1 << 17)) == 0; /* ditto kernel */ + hw->exclude_hv = 1; /* ditto hypervisor */ + hw->wakeup_events = 1; /* wakeup every n events */ + dump_perf_event_attr (hw); +} + +static int +start_one_ctr (int ii, size_t pgsz, hdrv_pcl_ctx_t * pctx, char *error_string) +{ + // pe_attr should have been initialized in hwcdrv_create_counters() + struct perf_event_attr pe_attr; + memcpy (&pe_attr, &global_perf_event_def[ii].hw, sizeof (pe_attr)); + + // but we adjust the period, so make sure that pctx->ctr_list[ii].last_overflow_period has been set + pe_attr.sample_period = pctx->ctr_list[ii].last_overflow_period; + + int hwc_fd = perf_event_open (&pe_attr, pctx->tid, -1, -1, 0); + if (hwc_fd == -1) + { + TprintfT (DBG_LT1, "%s idx=%d perf_event_open failed, errno=%d\n", + error_string, ii, errno); + return 1; + } + + size_t buffer_area_sz = (NPAGES_PER_BUF + 1) * pgsz; // add a page for metadata + void * buf = mmap (NULL, buffer_area_sz, //YXXX is this a safe call? + PROT_READ | PROT_WRITE, MAP_SHARED, hwc_fd, 0); + if (buf == MAP_FAILED) + { + TprintfT (0, "sz = %ld, pgsz = %ld\n err=%s idx=%d mmap failed: %s\n", + (long) buffer_area_sz, (long) pgsz, error_string, ii, strerror (errno)); + return 1; + } + pctx->ctr_list[ii].ev_def = &global_perf_event_def[ii]; // why do we set ev_def? we never seem to use it + pctx->ctr_list[ii].fd = hwc_fd; + pctx->ctr_list[ii].buf_state.buf = buf; + pctx->ctr_list[ii].buf_state.pagesz = pgsz; + pctx->ctr_list[ii].value_state.prev_ena_ts = 0; + pctx->ctr_list[ii].value_state.prev_run_ts = 0; + pctx->ctr_list[ii].value_state.prev_value = 0; + pctx->ctr_list[ii].last_overflow_time = gethrtime (); + + /* set async mode */ + long flags = fcntl (hwc_fd, F_GETFL, 0) | O_ASYNC; + int rc = fcntl (hwc_fd, F_SETFL, flags); + if (rc == -1) + { + TprintfT (0, "%s idx=%d O_ASYNC failed\n", error_string, ii); + return 1; + } + + /* + * set lwp ownership of the fd + * See BUGS section of "man perf_event_open": + * The F_SETOWN_EX option to fcntl(2) is needed to properly get + * overflow signals in threads. This was introduced in Linux 2.6.32. + * Legacy references: + * see http://lkml.org/lkml/2009/8/4/128 + * google man fcntl F_SETOWN_EX -conflict + * "From Linux 2.6.32 onward, use F_SETOWN_EX to target + * SIGIO and SIGURG signals at a particular thread." + * http://icl.cs.utk.edu/papi/docs/da/d2a/examples__v2_8x_2self__smpl__multi_8c.html + * See 2010 CSCADS presentation by Eranian + */ + struct f_owner_ex fowner_ex; + fowner_ex.type = F_OWNER_TID; + fowner_ex.pid = pctx->tid; + rc = fcntl (hwc_fd, F_SETOWN_EX, (unsigned long) &fowner_ex); + if (rc == -1) + { + TprintfT (0, "%s idx=%d F_SETOWN failed\n", error_string, ii); + return 1; + } + + /* Use sigio so handler can determine FD via siginfo->si_fd. */ + rc = fcntl (hwc_fd, F_SETSIG, SIGIO); + if (rc == -1) + { + TprintfT (0, "%s idx=%d F_SETSIG failed\n", error_string, ii); + return 1; + } + return 0; +} + +static int +stop_one_ctr (int ii, counter_state_t *ctr_list) +{ + int hwc_rc = 0; + if (-1 == ioctl (ctr_list[ii].fd, PERF_EVENT_IOC_DISABLE, 1)) + { + TprintfT (0, "hwcdrv: ERROR: PERF_EVENT_IOC_DISABLE #%d failed: errno=%d\n", ii, errno); + hwc_rc = HWCFUNCS_ERROR_GENERIC; + } + void *buf = ctr_list[ii].buf_state.buf; + if (buf) + { + size_t bufsz = (NPAGES_PER_BUF + 1) * ctr_list[ii].buf_state.pagesz; + ctr_list[ii].buf_state.buf = NULL; + int tmprc = munmap (buf, bufsz); + if (tmprc) + { + TprintfT (0, "hwcdrv: ERROR: munmap() #%d failed: errno=%d\n", ii, errno); + hwc_rc = HWCFUNCS_ERROR_GENERIC; + } + } + if (-1 == close (ctr_list[ii].fd)) + { + TprintfT (0, "hwcdrv: ERROR: close(fd) #%d failed: errno=%d\n", ii, errno); + hwc_rc = HWCFUNCS_ERROR_GENERIC; + } + return hwc_rc; +} + +/* HWCDRV_API for thread-specific actions */ +HWCDRV_API int +hwcdrv_lwp_init (void) +{ + return hwcdrv_start (); +} + +HWCDRV_API void +hwcdrv_lwp_fini (void) +{ + hwcdrv_free_counters (); /* also sets pctx->ctr_list=NULL; */ +} + +/* open */ +static int +hdrv_pcl_internal_open () +{ + if (hdrv_pcl_state.internal_open_called) + { + TprintfT (0, "hwcdrv: WARNING: hdrv_pcl_internal_open: already called\n"); + return HWCFUNCS_ERROR_ALREADY_CALLED; + } + + // determine if PCL is available + perf_event_def_t tmp_event_def; + memset (&tmp_event_def, 0, sizeof (tmp_event_def)); + struct perf_event_attr *pe_attr = &tmp_event_def.hw; + init_perf_event (pe_attr, 0, 0); + pe_attr->type = PERF_TYPE_HARDWARE; // specify abstracted HW event + pe_attr->config = PERF_COUNT_HW_INSTRUCTIONS; // specify abstracted insts + int hwc_fd = perf_event_open (pe_attr, + 0, // pid/tid, 0 is self + -1, // cpu, -1 is per-thread mode + -1, // group_fd, -1 is root + 0); // flags + if (hwc_fd == -1) + { + TprintfT (DBG_LT1, "hwcdrv: WARNING: hdrv_pcl_internal_open:" + " perf_event_open() failed, errno=%d\n", errno); + goto internal_open_error; + } + + /* see if the PCL is new enough to know about F_SETOWN_EX */ + struct f_owner_ex fowner_ex; + fowner_ex.type = F_OWNER_TID; + fowner_ex.pid = hwcdrv_gettid (); // "pid=tid" is correct w/F_OWNER_TID + if (fcntl (hwc_fd, F_SETOWN_EX, (unsigned long) &fowner_ex) == -1) + { + TprintfT (DBG_LT1, "hwcdrv: WARNING: hdrv_pcl_internal_open: " + "F_SETOWN failed, errno=%d\n", errno); + close (hwc_fd); + goto internal_open_error; + } + close (hwc_fd); + + hdrv_pcl_state.internal_open_called = 1; + hdrv_pcl_state.library_ok = 1; // set to non-zero to show it's initted + hdrv_pcl_about.cpcN_cpuver = CPUVER_UNDEFINED; + TprintfT (DBG_LT2, "hwcdrv: hdrv_pcl_internal_open()\n"); + for (int ii = 0; hdrv_pcbe_drivers[ii]; ii++) + { + hdrv_pcbe_api_t *ppcbe = hdrv_pcbe_drivers[ii]; + if (!ppcbe->hdrv_pcbe_init ()) + { + hdrv_pcl_about.cpcN_cciname = ppcbe->hdrv_pcbe_impl_name (); + hdrv_pcl_about.cpcN_cpuver = hwcdrv_lookup_cpuver (hdrv_pcl_about.cpcN_cciname); + if (hdrv_pcl_about.cpcN_cpuver == CPUVER_UNDEFINED) + goto internal_open_error; + hdrv_pcl_about.cpcN_npics = ppcbe->hdrv_pcbe_ncounters (); + hdrv_pcl_about.cpcN_docref = ppcbe->hdrv_pcbe_cpuref (); + hdrv_pcl_state.get_events = ppcbe->hdrv_pcbe_get_events; + hwcdrv_get_x86_eventnum = ppcbe->hdrv_pcbe_get_eventnum; + break; + } + } + if (hdrv_pcl_about.cpcN_npics > MAX_PICS) + { + TprintfT (0, "hwcdrv: WARNING: hdrv_pcl_internal_open:" + " reducing number of HWCs from %u to %u on processor '%s'\n", + hdrv_pcl_about.cpcN_npics, MAX_PICS, hdrv_pcl_about.cpcN_cciname); + hdrv_pcl_about.cpcN_npics = MAX_PICS; + } + TprintfT (DBG_LT1, "hwcdrv: hdrv_pcl_internal_open:" + " perf_event cpuver=%d, name='%s'\n", + hdrv_pcl_about.cpcN_cpuver, hdrv_pcl_about.cpcN_cciname); + return 0; + +internal_open_error: + hdrv_pcl_about.cpcN_cpuver = CPUVER_UNDEFINED; + hdrv_pcl_about.cpcN_npics = 0; + hdrv_pcl_about.cpcN_docref = NULL; + hdrv_pcl_about.cpcN_cciname = NULL; + return HWCFUNCS_ERROR_NOT_SUPPORTED; +} + +static void * +single_thread_tsd_ftn () +{ + static hdrv_pcl_ctx_t tsd_context; + return &tsd_context; +} + +/* HWCDRV_API */ +HWCDRV_API int +hwcdrv_init (hwcfuncs_abort_fn_t abort_ftn, int *tsd_sz) +{ + hdrv_pcl_state.find_vpc_ctx = single_thread_tsd_ftn; + if (tsd_sz) + *tsd_sz = sizeof (hdrv_pcl_ctx_t); + + if (hdrv_pcl_state.internal_open_called) + return HWCFUNCS_ERROR_ALREADY_CALLED; + return hdrv_pcl_internal_open (); +} + +HWCDRV_API void +hwcdrv_get_info (int *cpuver, const char **cciname, uint_t *npics, + const char **docref, uint64_t *support) +{ + if (cpuver) + *cpuver = hdrv_pcl_about.cpcN_cpuver; + if (cciname) + *cciname = hdrv_pcl_about.cpcN_cciname; + if (npics) + *npics = hdrv_pcl_about.cpcN_npics; + if (docref) + *docref = hdrv_pcl_about.cpcN_docref; + if (support) + *support = HWCFUNCS_SUPPORT_OVERFLOW_PROFILING | HWCFUNCS_SUPPORT_OVERFLOW_CTR_ID; +} + +HWCDRV_API int +hwcdrv_enable_mt (hwcfuncs_tsd_get_fn_t tsd_ftn) +{ + if (tsd_ftn) + hdrv_pcl_state.find_vpc_ctx = tsd_ftn; + else + { + TprintfT (0, "hwcdrv: ERROR: enable_mt(): tsd_ftn==NULL\n"); + return HWCFUNCS_ERROR_UNAVAIL; + } + return 0; +} + +HWCDRV_API int +hwcdrv_get_descriptions (hwcf_hwc_cb_t *hwc_cb, hwcf_attr_cb_t *attr_cb) +{ + int count = 0; + if (hwc_cb && hdrv_pcl_state.get_events) + count = hdrv_pcl_state.get_events (hwc_cb); + if (attr_cb) + for (int ii = 0; perfctr_attrs_table && perfctr_attrs_table[ii].attrname; ii++) + attr_cb (perfctr_attrs_table[ii].attrname); + if (!count) + return -1; + return 0; +} + +HWCDRV_API int +hwcdrv_assign_regnos (Hwcentry* entries[], unsigned numctrs) +{ + return hwcdrv_assign_all_regnos (entries, numctrs); +} + +static int +internal_hwc_start (int fd) +{ + int rc = ioctl (fd, PERF_EVENT_IOC_REFRESH, 1); + if (rc == -1) + { + TprintfT (DBG_LT0, "hwcdrv: ERROR: internal_hwc_start:" + " PERF_EVENT_IOC_REFRESH(fd=%d) failed: errno=%d\n", fd, errno); + return HWCFUNCS_ERROR_UNAVAIL; + } + TprintfT (DBG_LT3, "hwcdrv: internal_hwc_start(fd=%d)\n", fd); + return 0; +} + +HWCDRV_API int +hwcdrv_overflow (siginfo_t *si, hwc_event_t *eventp, hwc_event_t *lost_events) +{ + /* set expired counters to overflow value and all others to 0 */ + /* return 0: OK, counters should be restarted */ + /* return non-zero: eventp not set, counters should not be restarted */ + /* clear return values */ + int ii; + for (ii = 0; ii < hdrv_pcl_state.hwcdef_cnt; ii++) + { + eventp->ce_pic[ii] = 0; + lost_events->ce_pic[ii] = 0; + } + hrtime_t sig_ts = gethrtime (); //YXXX get this from HWC event? + eventp->ce_hrt = sig_ts; + lost_events->ce_hrt = sig_ts; + + /* determine source signal */ + int signal_fd = -1; + switch (si->si_code) + { + case POLL_HUP: /* expected value from pcl */ + /* According to Stephane Eranian: + * "expect POLL_HUP instead of POLL_IN because we are + * in one-shot mode (IOC_REFRESH)" + */ + signal_fd = si->si_fd; + break; + case SI_TKILL: /* event forwarded by tkill */ + /* DBX can only forward SI_TKILL when it detects POLL_HUP + * unfortunately, this means that si->si_fd has been lost... + * We need to process the buffers, but we don't know the fd! + */ + TprintfT (DBG_LT0, "hwcdrv: sig_ts=%llu: WARNING: hwcdrv_overflow:" + " SI_TKILL detected\n", sig_ts); + break; + default: + // "sometimes we see a POLL_IN (1) with very high event rates," + // according to eranian(?) + TprintfT (DBG_LT0, "hwcdrv: sig_ts=%llu: ERROR: hwcdrv_overflow:" + " unexpected si_code 0x%x\n", sig_ts, si->si_code); + return HWCFUNCS_ERROR_GENERIC; + } + + hdrv_pcl_ctx_t * pctx = hdrv_pcl_state.find_vpc_ctx (); + if (!pctx) + { + TprintfT (DBG_LT0, "hwcdrv: sig_ts=%llu: ERROR: hwcdrv_overflow:" + " tsd context is NULL\n", sig_ts); + return HWCFUNCS_ERROR_UNEXPECTED; + } + counter_state_t * ctr_list = (counter_state_t *) pctx->ctr_list; + if (!ctr_list) + { + TprintfT (DBG_LT0, "hwcdrv: sig_ts=%llu: WARNING: hwcdrv_overflow:" + " ctr_list is NULL\n", sig_ts); + return HWCFUNCS_ERROR_UNEXPECTED; + } + + /* clear needs_restart flag */ + for (ii = 0; ii < hdrv_pcl_state.hwcdef_cnt; ii++) + ctr_list[ii].needs_restart = 0; + + /* attempt to identify the counter to read */ + int signal_idx = -1; + pctx->signal_fd = signal_fd; // save the signal provided by siginfo_t + if (signal_fd != -1) + { + for (ii = 0; ii < hdrv_pcl_state.hwcdef_cnt; ii++) + { + if (ctr_list[ii].fd == signal_fd) + { + signal_idx = ii; + break; + } + } + } + + if (signal_idx < 0) + { + TprintfT (DBG_LT0, "hwcdrv: sig_ts=%llu: ERROR: hwcdrv_overflow:" + " pmc not determined!\n", sig_ts); + lost_events->ce_pic[0] = 1; /* record a bogus value into experiment */ + // note: bogus value may get overwritten in loop below + } + + /* capture sample(s). In addition to signal_idx, check other counters. */ + struct perf_event_header sheader; + int idx; + for (idx = 0; idx < hdrv_pcl_state.hwcdef_cnt; idx++) + { + int num_recs = 0; + while (1) + { + /* check for samples */ + struct perf_event_mmap_page *metadata = ctr_list[idx].buf_state.buf; + if (metadata == NULL) + break; // empty + if (metadata->data_tail == metadata->data_head) + break; // empty + + /* read header */ + if (read_buf (&ctr_list[idx].buf_state, &sheader, sizeof (sheader))) + break; + num_recs++; + + /* check for PERF_RECORD_SAMPLE */ + size_t datasz = sheader.size - sizeof (struct perf_event_header); + if (sheader.type != PERF_RECORD_SAMPLE) + { + TprintfT (DBG_LT2, "hwcdrv: sig_ts=%llu: WARNING: hwcdrv_overflow:" + " unexpected recd type=%d\n", + sig_ts, sheader.type); + if (skip_buf (&ctr_list[idx].buf_state, datasz)) + { + TprintfT (DBG_LT0, "hwcdrv: sig_ts=%llu: ERROR: hwcdrv_overflow:" + " skip recd type=%d failed\n", sig_ts, sheader.type); + lost_events->ce_pic[idx] = 4; /* record a bogus value */ + break; // failed to skip buffer?? + } + lost_events->ce_pic[idx] = 2; /* record a bogus value */ + continue; // advance to next record + } + + /* type is PERF_RECORD_SAMPLE */ + uint64_t value, lostv; + if (read_sample (&ctr_list[idx], datasz, &value, &lostv)) + { + TprintfT (DBG_LT0, "hwcdrv: sig_ts=%llu: ERROR: hwcdrv_overflow:" + " read_sample() failed\n", sig_ts); + lost_events->ce_pic[idx] = 3; // record a bogus value + break; // failed to read sample data?? + } + TprintfT (DBG_LT3, "hwcdrv: sig_ts=%llu: hwcdrv_overflow:" + " idx=%d value=%llu lost=%llu\n", (unsigned long long) sig_ts, + idx, (unsigned long long) value, (unsigned long long) lostv); + if (eventp->ce_pic[idx]) + { + TprintfT (DBG_LT2, "hwcdrv: sig_ts=%llu: WARNING: hwcdrv_overflow:" + " idx=%d previous sample recorded as lost_event\n", sig_ts, idx); + lost_events->ce_pic[idx] += eventp->ce_pic[idx]; + } + eventp->ce_pic[idx] = value; + lost_events->ce_pic[idx] += lostv; + } + + /* debug output for unexpected (but common) cases */ + if (idx == signal_idx) + { + if (num_recs != 1) + TprintfT (DBG_LT2, "hwcdrv: sig_ts=%llu: WARNING: hwcdrv_overflow:" + " %d records for signal_idx=%d\n", sig_ts, num_recs, signal_idx); + } + else if (num_recs) + TprintfT (DBG_LT2, "hwcdrv: sig_ts=%llu: WARNING: hwcdrv_overflow:" + " %d unexpected record(s) for idx=%d (signal_idx=%d)\n", + sig_ts, num_recs, idx, signal_idx); + + /* trigger counter restart whenever records were found */ + if (num_recs) + { + /* check whether to adapt the overflow interval */ + /* This is the Linux version. + * The Solaris version is in hwprofile.c collector_update_overflow_counters(). + */ + hrtime_t min_time = global_perf_event_def[idx].min_time; + if (min_time > 0 // overflow interval is adaptive + && sig_ts - ctr_list[idx].last_overflow_time < min_time) // last interval below min + { + /* pick a new overflow interval */ + /* roughly doubled, but add funny numbers */ + /* hopefully the result is prime or not a multiple of some # of ops/loop */ + uint64_t new_period = 2 * ctr_list[idx].last_overflow_period + 37; +#if 0 + // On Solaris, we report the adjustment to the log file. + // On Linux it's hard for us to do so since hwcdrv_pcl.c doesn't know about collector_interface, SP_JCMD_COMMENT, or COL_COMMENT_HWCADJ. + // For now we simply don't report. + collector_interface->writeLog ("<event kind=\"%s\" id=\"%d\">%s %d -> %d</event>\n", + SP_JCMD_COMMENT, COL_COMMENT_HWCADJ, global_perf_event_def[idx].name, + ctr_list[idx].last_overflow_period, new_period); +#endif + /* There are a variety of ways of resetting the period on Linux. + * The most elegant is + * ioctl(fd,PERF_EVENT_IOC_PERIOD,&period) + * but check the perf_event_open man page for PERF_EVENT_IOC_PERIOD: + * > Prior to Linux 2.6.36 this ioctl always failed due to a bug in the kernel. + * > Prior to Linux 3.14 (or 3.7 on ARM), the new period did not take effect + * until after the next overflow. + * So we're kind of stuck shutting the fd down and restarting it with the new period. + */ + if (stop_one_ctr (idx, ctr_list)) + { + // EUGENE figure out what to do on error + } + ctr_list[idx].last_overflow_period = new_period; + if (start_one_ctr (idx, ctr_list[idx].buf_state.pagesz, pctx, "hwcdrv: ERROR: hwcdrv_overflow (readjust overflow):")) + { + // EUGENE figure out what to do on error + } + } + ctr_list[idx].last_overflow_time = sig_ts; +#if 0 + ctr_list[idx].needs_restart = 1; +#else // seems to be more reliable to restart here instead of hwcdrv_sighlr_restart() + internal_hwc_start (ctr_list[idx].fd); +#endif + } + } + return 0; // OK to restart counters +} + +HWCDRV_API int +hwcdrv_sighlr_restart (const hwc_event_t *pp) +{ +#if 0 // restarting here doesn't seem to work as well as restarting in hwcdrv_overflow() + hdrv_pcl_ctx_t * pctx = hdrv_pcl_state.find_vpc_ctx (); + if (!pctx) + { + TprintfT (DBG_LT0, "hwcdrv: ERROR: hwcdrv_sighlr_restart: find_vpc_ctx()==NULL\n"); + return -1; + } + counter_state_t * ctr_list = (counter_state_t *) pctx->ctr_list; + if (!ctr_list) + { + TprintfT (DBG_LT0, "hwcdrv: WARNING: hwcdrv_sighlr_restart: ctr_list is NULL\n"); + return -1; + } + int errors = 0; + for (int ii = 0; ii < hdrv_pcl_state.hwcdef_cnt; ii++) + { + if (ctr_list[ii].needs_restart) + errors |= internal_hwc_start (ctr_list[ii].fd); + ctr_list[ii].needs_restart = 0; + } + return errors; +#else + return 0; +#endif +} + +/* create counters based on hwcdef[] */ +HWCDRV_API int +hwcdrv_create_counters (unsigned hwcdef_cnt, Hwcentry *hwcdef) +{ + if (hwcdef_cnt > hdrv_pcl_about.cpcN_npics) + { + logerr (GTXT ("More than %d counters were specified\n"), hdrv_pcl_about.cpcN_npics); /*!*/ + return HWCFUNCS_ERROR_HWCARGS; + } + if (hdrv_pcl_about.cpcN_cpuver == CPUVER_UNDEFINED) + { + logerr (GTXT ("Processor not supported\n")); + return HWCFUNCS_ERROR_HWCARGS; + } + + /* add counters */ + for (unsigned idx = 0; idx < hwcdef_cnt; idx++) + { + perf_event_def_t *glb_event_def = &global_perf_event_def[idx]; + memset (glb_event_def, 0, sizeof (perf_event_def_t)); + unsigned int pmc_sel; + eventsel_t evntsel; + if (hwcfuncs_get_x86_eventsel (hwcdef[idx].reg_num, + hwcdef[idx].int_name, &evntsel, &pmc_sel)) + { + TprintfT (0, "hwcdrv: ERROR: hwcfuncs_get_x86_eventsel() failed\n"); + return HWCFUNCS_ERROR_HWCARGS; + } + glb_event_def->reg_num = pmc_sel; + glb_event_def->eventsel = evntsel; + glb_event_def->counter_preload = hwcdef[idx].val; + glb_event_def->min_time = hwcdef[idx].min_time; + glb_event_def->name = strdup (hwcdef[idx].name); // memory leak??? very minor + init_perf_event (&glb_event_def->hw, glb_event_def->eventsel, + glb_event_def->counter_preload); + TprintfT (DBG_LT1, "hwcdrv: create_counters: pic=%u name='%s' interval=%lld" + "(min_time=%lld): reg_num=0x%x eventsel=0x%llx ireset=%lld usr=%lld sys=%lld\n", + idx, hwcdef[idx].int_name, (long long) glb_event_def->counter_preload, + (long long) glb_event_def->min_time, (int) glb_event_def->reg_num, + (long long) glb_event_def->eventsel, + (long long) HW_INTERVAL_PRESET (hwcdef[idx].val), + (long long) glb_event_def->hw.exclude_user, + (long long) glb_event_def->hw.exclude_kernel); + } + + hdrv_pcl_state.hwcdef_cnt = hwcdef_cnt; + return 0; +} + +HWCDRV_API int +hwcdrv_free_counters () // note: only performs shutdown for this thread +{ + hdrv_pcl_ctx_t * pctx; + if (!COUNTERS_ENABLED ()) + return 0; + pctx = hdrv_pcl_state.find_vpc_ctx (); + if (!pctx) + { + TprintfT (0, "hwcdrv: WARNING: hwcdrv_free_counters: tsd context is NULL\n"); + return HWCFUNCS_ERROR_GENERIC; + } + counter_state_t *ctr_list = pctx->ctr_list; + if (!ctr_list) + { + // fork child: prolog suspends hwcs, then epilog frees them + TprintfT (DBG_LT1, "hwcdrv: WARNING: hwcdrv_free_counters: ctr_list is already NULL\n"); + return 0; + } + int hwc_rc = 0; + for (int ii = 0; ii < hdrv_pcl_state.hwcdef_cnt; ii++) + if (stop_one_ctr (ii, ctr_list)) + hwc_rc = HWCFUNCS_ERROR_GENERIC; + TprintfT (DBG_LT1, "hwcdrv: hwcdrv_free_counters(tid=0x%lx).\n", pctx->tid); + pctx->ctr_list = NULL; + return hwc_rc; +} + +HWCDRV_API int +hwcdrv_start (void) /* must be called from each thread ? */ +{ + hdrv_pcl_ctx_t *pctx = NULL; + if (!COUNTERS_ENABLED ()) + { + TprintfT (DBG_LT1, "hwcdrv: WARNING: hwcdrv_start: no counters to start \n"); + return 0; + } + if (!hdrv_pcl_state.library_ok) + { + TprintfT (0, "hwcdrv: ERROR: hwcdrv_start: library is not open\n"); + return HWCFUNCS_ERROR_NOT_SUPPORTED; + } + + /* + * set up per-thread context + */ + pctx = hdrv_pcl_state.find_vpc_ctx (); + if (!pctx) + { + TprintfT (0, "hwcdrv: ERROR: hwcdrv_start: tsd context is NULL\n"); + return HWCFUNCS_ERROR_UNEXPECTED; + } + pctx->tid = hwcdrv_gettid (); + TprintfT (DBG_LT1, "hwcdrv: hwcdrv_start(tid=0x%lx)\n", pctx->tid); + + /* + * create per-thread counter list + */ + counter_state_t *ctr_list = (counter_state_t *) calloc (hdrv_pcl_state.hwcdef_cnt, + sizeof (counter_state_t)); + if (!ctr_list) + { + TprintfT (0, "hwcdrv: ERROR: hwcdrv_start: calloc(ctr_list) failed\n"); + return HWCFUNCS_ERROR_MEMORY; + } + int ii; + for (ii = 0; ii < hdrv_pcl_state.hwcdef_cnt; ii++) + ctr_list[ii].fd = -1; // invalidate fds in case we have to close prematurely + pctx->ctr_list = ctr_list; + + /* + * bind the counters + */ + size_t pgsz = sysconf (_SC_PAGESIZE); + for (ii = 0; ii < hdrv_pcl_state.hwcdef_cnt; ii++) + { + ctr_list[ii].last_overflow_period = global_perf_event_def[ii].hw.sample_period; + if (start_one_ctr (ii, pgsz, pctx, "hwcdrv: ERROR: hwcdrv_start:")) goto hwcdrv_start_cleanup; + } + + /* + * start the counters + */ + for (ii = 0; ii < hdrv_pcl_state.hwcdef_cnt; ii++) + { + int rc = internal_hwc_start (ctr_list[ii].fd); + if (rc < 0) + goto hwcdrv_start_cleanup; + } + return 0; + +hwcdrv_start_cleanup: + hwcdrv_free_counters (); // PERF_EVENT_IOC_DISABLE and close() for all fds + return HWCFUNCS_ERROR_UNAVAIL; +} + +HWCDRV_API int +hwcdrv_lwp_suspend (void) /* must be called from each thread */ +{ + if (!COUNTERS_ENABLED ()) + { + TprintfT (DBG_LT1, "hwcdrv: WARNING: hwcdrv_lwp_suspend: no counters\n"); + return 0; + } + TprintfT (DBG_LT1, "hwcdrv: hwcdrv_lwp_suspend()\n"); + return hwcdrv_free_counters (); +} + +HWCDRV_API int +hwcdrv_lwp_resume (void) /* must be called from each thread */ +{ + if (!COUNTERS_ENABLED ()) + { + TprintfT (DBG_LT1, "hwcdrv: WARNING: hwcdrv_lwp_resume: no counters\n"); + return 0; + } + TprintfT (DBG_LT1, "hwcdrv: hwcdrv_lwp_resume()\n"); + return hwcdrv_start (); +} + +HWCDRV_API int +hwcdrv_read_events (hwc_event_t *overflow_data, hwc_event_samples_t *sampled_data) +{ + overflow_data->ce_hrt = 0; + for (int i = 0; i < MAX_PICS; i++) + { + overflow_data->ce_pic[i] = 0; + if (sampled_data) + HWCFUNCS_SAMPLE_RESET (&sampled_data->sample[i]); + } + return 0; +} + +/*---------------------------------------------------------------------------*/ +/* HWCDRV_API */ + +hwcdrv_api_t hwcdrv_pcl_api = { + hwcdrv_init, + hwcdrv_get_info, + hwcdrv_enable_mt, + hwcdrv_get_descriptions, + hwcdrv_assign_regnos, + hwcdrv_create_counters, + hwcdrv_start, + hwcdrv_overflow, + hwcdrv_read_events, + hwcdrv_sighlr_restart, + hwcdrv_lwp_suspend, + hwcdrv_lwp_resume, + hwcdrv_free_counters, + hwcdrv_lwp_init, + hwcdrv_lwp_fini, + -1 // hwcdrv_init_status +}; |