/* * Copyright 2014 Tilera Corporation. All Rights Reserved. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License * as published by the Free Software Foundation, version 2. * * This program is distributed in the hope that it will be useful, but * WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or * NON INFRINGEMENT. See the GNU General Public License for * more details. * * * Perf_events support for Tile processor. * * This code is based upon the x86 perf event * code, which is: * * Copyright (C) 2008 Thomas Gleixner * Copyright (C) 2008-2009 Red Hat, Inc., Ingo Molnar * Copyright (C) 2009 Jaswinder Singh Rajput * Copyright (C) 2009 Advanced Micro Devices, Inc., Robert Richter * Copyright (C) 2008-2009 Red Hat, Inc., Peter Zijlstra * Copyright (C) 2009 Intel Corporation, * Copyright (C) 2009 Google, Inc., Stephane Eranian */ #include #include #include #include #include #include #include #include #include #include #include #include #include #define TILE_MAX_COUNTERS 4 #define PERF_COUNT_0_IDX 0 #define PERF_COUNT_1_IDX 1 #define AUX_PERF_COUNT_0_IDX 2 #define AUX_PERF_COUNT_1_IDX 3 struct cpu_hw_events { int n_events; struct perf_event *events[TILE_MAX_COUNTERS]; /* counter order */ struct perf_event *event_list[TILE_MAX_COUNTERS]; /* enabled order */ int assign[TILE_MAX_COUNTERS]; unsigned long active_mask[BITS_TO_LONGS(TILE_MAX_COUNTERS)]; unsigned long used_mask; }; /* TILE arch specific performance monitor unit */ struct tile_pmu { const char *name; int version; const int *hw_events; /* generic hw events table */ /* generic hw cache events table */ const int (*cache_events)[PERF_COUNT_HW_CACHE_MAX] [PERF_COUNT_HW_CACHE_OP_MAX] [PERF_COUNT_HW_CACHE_RESULT_MAX]; int (*map_hw_event)(u64); /*method used to map hw events */ int (*map_cache_event)(u64); /*method used to map cache events */ u64 max_period; /* max sampling period */ u64 cntval_mask; /* counter width mask */ int cntval_bits; /* counter width */ int max_events; /* max generic hw events in map */ int num_counters; /* number base + aux counters */ int num_base_counters; /* number base counters */ }; DEFINE_PER_CPU(u64, perf_irqs); static DEFINE_PER_CPU(struct cpu_hw_events, cpu_hw_events); #define TILE_OP_UNSUPP (-1) #ifndef __tilegx__ /* TILEPro hardware events map */ static const int tile_hw_event_map[] = { [PERF_COUNT_HW_CPU_CYCLES] = 0x01, /* ONE */ [PERF_COUNT_HW_INSTRUCTIONS] = 0x06, /* MP_BUNDLE_RETIRED */ [PERF_COUNT_HW_CACHE_REFERENCES] = TILE_OP_UNSUPP, [PERF_COUNT_HW_CACHE_MISSES] = TILE_OP_UNSUPP, [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = 0x16, /* MP_CONDITIONAL_BRANCH_ISSUED */ [PERF_COUNT_HW_BRANCH_MISSES] = 0x14, /* MP_CONDITIONAL_BRANCH_MISSPREDICT */ [PERF_COUNT_HW_BUS_CYCLES] = TILE_OP_UNSUPP, }; #else /* TILEGx hardware events map */ static const int tile_hw_event_map[] = { [PERF_COUNT_HW_CPU_CYCLES] = 0x181, /* ONE */ [PERF_COUNT_HW_INSTRUCTIONS] = 0xdb, /* INSTRUCTION_BUNDLE */ [PERF_COUNT_HW_CACHE_REFERENCES] = TILE_OP_UNSUPP, [PERF_COUNT_HW_CACHE_MISSES] = TILE_OP_UNSUPP, [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = 0xd9, /* COND_BRANCH_PRED_CORRECT */ [PERF_COUNT_HW_BRANCH_MISSES] = 0xda, /* COND_BRANCH_PRED_INCORRECT */ [PERF_COUNT_HW_BUS_CYCLES] = TILE_OP_UNSUPP, }; #endif #define C(x) PERF_COUNT_HW_CACHE_##x /* * Generalized hw caching related hw_event table, filled * in on a per model basis. A value of -1 means * 'not supported', any other value means the * raw hw_event ID. */ #ifndef __tilegx__ /* TILEPro hardware cache event map */ static const int tile_cache_event_map[PERF_COUNT_HW_CACHE_MAX] [PERF_COUNT_HW_CACHE_OP_MAX] [PERF_COUNT_HW_CACHE_RESULT_MAX] = { [C(L1D)] = { [C(OP_READ)] = { [C(RESULT_ACCESS)] = TILE_OP_UNSUPP, [C(RESULT_MISS)] = 0x21, /* RD_MISS */ }, [C(OP_WRITE)] = { [C(RESULT_ACCESS)] = TILE_OP_UNSUPP, [C(RESULT_MISS)] = 0x22, /* WR_MISS */ }, [C(OP_PREFETCH)] = { [C(RESULT_ACCESS)] = TILE_OP_UNSUPP, [C(RESULT_MISS)] = TILE_OP_UNSUPP, }, }, [C(L1I)] = { [C(OP_READ)] = { [C(RESULT_ACCESS)] = 0x12, /* MP_ICACHE_HIT_ISSUED */ [C(RESULT_MISS)] = TILE_OP_UNSUPP, }, [C(OP_WRITE)] = { [C(RESULT_ACCESS)] = TILE_OP_UNSUPP, [C(RESULT_MISS)] = TILE_OP_UNSUPP, }, [C(OP_PREFETCH)] = { [C(RESULT_ACCESS)] = TILE_OP_UNSUPP, [C(RESULT_MISS)] = TILE_OP_UNSUPP, }, }, [C(LL)] = { [C(OP_READ)] = { [C(RESULT_ACCESS)] = TILE_OP_UNSUPP, [C(RESULT_MISS)] = TILE_OP_UNSUPP, }, [C(OP_WRITE)] = { [C(RESULT_ACCESS)] = TILE_OP_UNSUPP, [C(RESULT_MISS)] = TILE_OP_UNSUPP, }, [C(OP_PREFETCH)] = { [C(RESULT_ACCESS)] = TILE_OP_UNSUPP, [C(RESULT_MISS)] = TILE_OP_UNSUPP, }, }, [C(DTLB)] = { [C(OP_READ)] = { [C(RESULT_ACCESS)] = 0x1d, /* TLB_CNT */ [C(RESULT_MISS)] = 0x20, /* TLB_EXCEPTION */ }, [C(OP_WRITE)] = { [C(RESULT_ACCESS)] = TILE_OP_UNSUPP, [C(RESULT_MISS)] = TILE_OP_UNSUPP, }, [C(OP_PREFETCH)] = { [C(RESULT_ACCESS)] = TILE_OP_UNSUPP, [C(RESULT_MISS)] = TILE_OP_UNSUPP, }, }, [C(ITLB)] = { [C(OP_READ)] = { [C(RESULT_ACCESS)] = 0x13, /* MP_ITLB_HIT_ISSUED */ [C(RESULT_MISS)] = TILE_OP_UNSUPP, }, [C(OP_WRITE)] = { [C(RESULT_ACCESS)] = TILE_OP_UNSUPP, [C(RESULT_MISS)] = TILE_OP_UNSUPP, }, [C(OP_PREFETCH)] = { [C(RESULT_ACCESS)] = TILE_OP_UNSUPP, [C(RESULT_MISS)] = TILE_OP_UNSUPP, }, }, [C(BPU)] = { [C(OP_READ)] = { [C(RESULT_ACCESS)] = TILE_OP_UNSUPP, [C(RESULT_MISS)] = TILE_OP_UNSUPP, }, [C(OP_WRITE)] = { [C(RESULT_ACCESS)] = TILE_OP_UNSUPP, [C(RESULT_MISS)] = TILE_OP_UNSUPP, }, [C(OP_PREFETCH)] = { [C(RESULT_ACCESS)] = TILE_OP_UNSUPP, [C(RESULT_MISS)] = TILE_OP_UNSUPP, }, }, }; #else /* TILEGx hardware events map */ static const int tile_cache_event_map[PERF_COUNT_HW_CACHE_MAX] [PERF_COUNT_HW_CACHE_OP_MAX] [PERF_COUNT_HW_CACHE_RESULT_MAX] = { [C(L1D)] = { /* * Like some other architectures (e.g. ARM), the performance * counters don't differentiate between read and write * accesses/misses, so this isn't strictly correct, but it's the * best we can do. Writes and reads get combined. */ [C(OP_READ)] = { [C(RESULT_ACCESS)] = TILE_OP_UNSUPP, [C(RESULT_MISS)] = 0x44, /* RD_MISS */ }, [C(OP_WRITE)] = { [C(RESULT_ACCESS)] = TILE_OP_UNSUPP, [C(RESULT_MISS)] = 0x45, /* WR_MISS */ }, [C(OP_PREFETCH)] = { [C(RESULT_ACCESS)] = TILE_OP_UNSUPP, [C(RESULT_MISS)] = TILE_OP_UNSUPP, }, }, [C(L1I)] = { [C(OP_READ)] = { [C(RESULT_ACCESS)] = TILE_OP_UNSUPP, [C(RESULT_MISS)] = TILE_OP_UNSUPP, }, [C(OP_WRITE)] = { [C(RESULT_ACCESS)] = TILE_OP_UNSUPP, [C(RESULT_MISS)] = TILE_OP_UNSUPP, }, [C(OP_PREFETCH)] = { [C(RESULT_ACCESS)] = TILE_OP_UNSUPP, [C(RESULT_MISS)] = TILE_OP_UNSUPP, }, }, [C(LL)] = { [C(OP_READ)] = { [C(RESULT_ACCESS)] = TILE_OP_UNSUPP, [C(RESULT_MISS)] = TILE_OP_UNSUPP, }, [C(OP_WRITE)] = { [C(RESULT_ACCESS)] = TILE_OP_UNSUPP, [C(RESULT_MISS)] = TILE_OP_UNSUPP, }, [C(OP_PREFETCH)] = { [C(RESULT_ACCESS)] = TILE_OP_UNSUPP, [C(RESULT_MISS)] = TILE_OP_UNSUPP, }, }, [C(DTLB)] = { [C(OP_READ)] = { [C(RESULT_ACCESS)] = 0x40, /* TLB_CNT */ [C(RESULT_MISS)] = 0x43, /* TLB_EXCEPTION */ }, [C(OP_WRITE)] = { [C(RESULT_ACCESS)] = 0x40, /* TLB_CNT */ [C(RESULT_MISS)] = 0x43, /* TLB_EXCEPTION */ }, [C(OP_PREFETCH)] = { [C(RESULT_ACCESS)] = TILE_OP_UNSUPP, [C(RESULT_MISS)] = TILE_OP_UNSUPP, }, }, [C(ITLB)] = { [C(OP_READ)] = { [C(RESULT_ACCESS)] = TILE_OP_UNSUPP, [C(RESULT_MISS)] = 0xd4, /* ITLB_MISS_INT */ }, [C(OP_WRITE)] = { [C(RESULT_ACCESS)] = TILE_OP_UNSUPP, [C(RESULT_MISS)] = 0xd4, /* ITLB_MISS_INT */ }, [C(OP_PREFETCH)] = { [C(RESULT_ACCESS)] = TILE_OP_UNSUPP, [C(RESULT_MISS)] = TILE_OP_UNSUPP, }, }, [C(BPU)] = { [C(OP_READ)] = { [C(RESULT_ACCESS)] = TILE_OP_UNSUPP, [C(RESULT_MISS)] = TILE_OP_UNSUPP, }, [C(OP_WRITE)] = { [C(RESULT_ACCESS)] = TILE_OP_UNSUPP, [C(RESULT_MISS)] = TILE_OP_UNSUPP, }, [C(OP_PREFETCH)] = { [C(RESULT_ACCESS)] = TILE_OP_UNSUPP, [C(RESULT_MISS)] = TILE_OP_UNSUPP, }, }, }; #endif static atomic_t tile_active_events; static DEFINE_MUTEX(perf_intr_reserve_mutex); static int tile_map_hw_event(u64 config); static int tile_map_cache_event(u64 config); static int tile_pmu_handle_irq(struct pt_regs *regs, int fault); /* * To avoid new_raw_count getting larger then pre_raw_count * in tile_perf_event_update(), we limit the value of max_period to 2^31 - 1. */ static const struct tile_pmu tilepmu = { #ifndef __tilegx__ .name = "tilepro", #else .name = "tilegx", #endif .max_events = ARRAY_SIZE(tile_hw_event_map), .map_hw_event = tile_map_hw_event, .hw_events = tile_hw_event_map, .map_cache_event = tile_map_cache_event, .cache_events = &tile_cache_event_map, .cntval_bits = 32, .cntval_mask = (1ULL << 32) - 1, .max_period = (1ULL << 31) - 1, .num_counters = TILE_MAX_COUNTERS, .num_base_counters = TILE_BASE_COUNTERS, }; static const struct tile_pmu *tile_pmu __read_mostly; /* * Check whether perf event is enabled. */ int tile_perf_enabled(void) { return atomic_read(&tile_active_events) != 0; } /* * Read Performance Counters. */ static inline u64 read_counter(int idx) { u64 val = 0; /* __insn_mfspr() only takes an immediate argument */ switch (idx) { case PERF_COUNT_0_IDX: val = __insn_mfspr(SPR_PERF_COUNT_0); break; case PERF_COUNT_1_IDX: val = __insn_mfspr(SPR_PERF_COUNT_1); break; case AUX_PERF_COUNT_0_IDX: val = __insn_mfspr(SPR_AUX_PERF_COUNT_0); break; case AUX_PERF_COUNT_1_IDX: val = __insn_mfspr(SPR_AUX_PERF_COUNT_1); break; default: WARN_ON_ONCE(idx > AUX_PERF_COUNT_1_IDX || idx < PERF_COUNT_0_IDX); } return val; } /* * Write Performance Counters. */ static inline void write_counter(int idx, u64 value) { /* __insn_mtspr() only takes an immediate argument */ switch (idx) { case PERF_COUNT_0_IDX: __insn_mtspr(SPR_PERF_COUNT_0, value); break; case PERF_COUNT_1_IDX: __insn_mtspr(SPR_PERF_COUNT_1, value); break; case AUX_PERF_COUNT_0_IDX: __insn_mtspr(SPR_AUX_PERF_COUNT_0, value); break; case AUX_PERF_COUNT_1_IDX: __insn_mtspr(SPR_AUX_PERF_COUNT_1, value); break; default: WARN_ON_ONCE(idx > AUX_PERF_COUNT_1_IDX || idx < PERF_COUNT_0_IDX); } } /* * Enable performance event by setting * Performance Counter Control registers. */ static inline void tile_pmu_enable_event(struct perf_event *event) { struct hw_perf_event *hwc = &event->hw; unsigned long cfg, mask; int shift, idx = hwc->idx; /* * prevent early activation from tile_pmu_start() in hw_perf_enable */ if (WARN_ON_ONCE(idx == -1)) return; if (idx < tile_pmu->num_base_counters) cfg = __insn_mfspr(SPR_PERF_COUNT_CTL); else cfg = __insn_mfspr(SPR_AUX_PERF_COUNT_CTL); switch (idx) { case PERF_COUNT_0_IDX: case AUX_PERF_COUNT_0_IDX: mask = TILE_EVENT_MASK; shift = 0; break; case PERF_COUNT_1_IDX: case AUX_PERF_COUNT_1_IDX: mask = TILE_EVENT_MASK << 16; shift = 16; break; default: WARN_ON_ONCE(idx < PERF_COUNT_0_IDX || idx > AUX_PERF_COUNT_1_IDX); return; } /* Clear mask bits to enable the event. */ cfg &= ~mask; cfg |= hwc->config << shift; if (idx < tile_pmu->num_base_counters) __insn_mtspr(SPR_PERF_COUNT_CTL, cfg); else __insn_mtspr(SPR_AUX_PERF_COUNT_CTL, cfg); } /* * Disable performance event by clearing * Performance Counter Control registers. */ static inline void tile_pmu_disable_event(struct perf_event *event) { struct hw_perf_event *hwc = &event->hw; unsigned long cfg, mask; int idx = hwc->idx; if (idx == -1) return; if (idx < tile_pmu->num_base_counters) cfg = __insn_mfspr(SPR_PERF_COUNT_CTL); else cfg = __insn_mfspr(SPR_AUX_PERF_COUNT_CTL); switch (idx) { case PERF_COUNT_0_IDX: case AUX_PERF_COUNT_0_IDX: mask = TILE_PLM_MASK; break; case PERF_COUNT_1_IDX: case AUX_PERF_COUNT_1_IDX: mask = TILE_PLM_MASK << 16; break; default: WARN_ON_ONCE(idx < PERF_COUNT_0_IDX || idx > AUX_PERF_COUNT_1_IDX); return; } /* Set mask bits to disable the event. */ cfg |= mask; if (idx < tile_pmu->num_base_counters) __insn_mtspr(SPR_PERF_COUNT_CTL, cfg); else __insn_mtspr(SPR_AUX_PERF_COUNT_CTL, cfg); } /* * Propagate event elapsed time into the generic event. * Can only be executed on the CPU where the event is active. * Returns the delta events processed. */ static u64 tile_perf_event_update(struct perf_event *event) { struct hw_perf_event *hwc = &event->hw; int shift = 64 - tile_pmu->cntval_bits; u64 prev_raw_count, new_raw_count; u64 oldval; int idx = hwc->idx; u64 delta; /* * Careful: an NMI might modify the previous event value. * * Our tactic to handle this is to first atomically read and * exchange a new raw count - then add that new-prev delta * count to the generic event atomically: */ again: prev_raw_count = local64_read(&hwc->prev_count); new_raw_count = read_counter(idx); oldval = local64_cmpxchg(&hwc->prev_count, prev_raw_count, new_raw_count); if (oldval != prev_raw_count) goto again; /* * Now we have the new raw value and have updated the prev * timestamp already. We can now calculate the elapsed delta * (event-)time and add that to the generic event. * * Careful, not all hw sign-extends above the physical width * of the count. */ delta = (new_raw_count << shift) - (prev_raw_count << shift); delta >>= shift; local64_add(delta, &event->count); local64_sub(delta, &hwc->period_left); return new_raw_count; } /* * Set the next IRQ period, based on the hwc->period_left value. * To be called with the event disabled in hw: */ static int tile_event_set_period(struct perf_event *event) { struct hw_perf_event *hwc = &event->hw; int idx = hwc->idx; s64 left = local64_read(&hwc->period_left); s64 period = hwc->sample_period; int ret = 0; /* * If we are way outside a reasonable range then just skip forward: */ if (unlikely(left <= -period)) { left = period; local64_set(&hwc->period_left, left); hwc->last_period = period; ret = 1; } if (unlikely(left <= 0)) { left += period; local64_set(&hwc->period_left, left); hwc->last_period = period; ret = 1; } if (left > tile_pmu->max_period) left = tile_pmu->max_period; /* * The hw event starts counting from this event offset, * mark it to be able to extra future deltas: */ local64_set(&hwc->prev_count, (u64)-left); write_counter(idx, (u64)(-left) & tile_pmu->cntval_mask); perf_event_update_userpage(event); return ret; } /* * Stop the event but do not release the PMU counter */ static void tile_pmu_stop(struct perf_event *event, int flags) { struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); struct hw_perf_event *hwc = &event->hw; int idx = hwc->idx; if (__test_and_clear_bit(idx, cpuc->active_mask)) { tile_pmu_disable_event(event); cpuc->events[hwc->idx] = NULL; WARN_ON_ONCE(hwc->state & PERF_HES_STOPPED); hwc->state |= PERF_HES_STOPPED; } if ((flags & PERF_EF_UPDATE) && !(hwc->state & PERF_HES_UPTODATE)) { /* * Drain the remaining delta count out of a event * that we are disabling: */ tile_perf_event_update(event); hwc->state |= PERF_HES_UPTODATE; } } /* * Start an event (without re-assigning counter) */ static void tile_pmu_start(struct perf_event *event, int flags) { struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); int idx = event->hw.idx; if (WARN_ON_ONCE(!(event->hw.state & PERF_HES_STOPPED))) return; if (WARN_ON_ONCE(idx == -1)) return; if (flags & PERF_EF_RELOAD) { WARN_ON_ONCE(!(event->hw.state & PERF_HES_UPTODATE)); tile_event_set_period(event); } event->hw.state = 0; cpuc->events[idx] = event; __set_bit(idx, cpuc->active_mask); unmask_pmc_interrupts(); tile_pmu_enable_event(event); perf_event_update_userpage(event); } /* * Add a single event to the PMU. * * The event is added to the group of enabled events * but only if it can be scehduled with existing events. */ static int tile_pmu_add(struct perf_event *event, int flags) { struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); struct hw_perf_event *hwc; unsigned long mask; int b, max_cnt; hwc = &event->hw; /* * We are full. */ if (cpuc->n_events == tile_pmu->num_counters) return -ENOSPC; cpuc->event_list[cpuc->n_events] = event; cpuc->n_events++; hwc->state = PERF_HES_UPTODATE | PERF_HES_STOPPED; if (!(flags & PERF_EF_START)) hwc->state |= PERF_HES_ARCH; /* * Find first empty counter. */ max_cnt = tile_pmu->num_counters; mask = ~cpuc->used_mask; /* Find next free counter. */ b = find_next_bit(&mask, max_cnt, 0); /* Should not happen. */ if (WARN_ON_ONCE(b == max_cnt)) return -ENOSPC; /* * Assign counter to event. */ event->hw.idx = b; __set_bit(b, &cpuc->used_mask); /* * Start if requested. */ if (flags & PERF_EF_START) tile_pmu_start(event, PERF_EF_RELOAD); return 0; } /* * Delete a single event from the PMU. * * The event is deleted from the group of enabled events. * If it is the last event, disable PMU interrupt. */ static void tile_pmu_del(struct perf_event *event, int flags) { struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); int i; /* * Remove event from list, compact list if necessary. */ for (i = 0; i < cpuc->n_events; i++) { if (cpuc->event_list[i] == event) { while (++i < cpuc->n_events) cpuc->event_list[i-1] = cpuc->event_list[i]; --cpuc->n_events; cpuc->events[event->hw.idx] = NULL; __clear_bit(event->hw.idx, &cpuc->used_mask); tile_pmu_stop(event, PERF_EF_UPDATE); break; } } /* * If there are no events left, then mask PMU interrupt. */ if (cpuc->n_events == 0) mask_pmc_interrupts(); perf_event_update_userpage(event); } /* * Propagate event elapsed time into the event. */ static inline void tile_pmu_read(struct perf_event *event) { tile_perf_event_update(event); } /* * Map generic events to Tile PMU. */ static int tile_map_hw_event(u64 config) { if (config >= tile_pmu->max_events) return -EINVAL; return tile_pmu->hw_events[config]; } /* * Map generic hardware cache events to Tile PMU. */ static int tile_map_cache_event(u64 config) { unsigned int cache_type, cache_op, cache_result; int code; if (!tile_pmu->cache_events) return -ENOENT; cache_type = (config >> 0) & 0xff; if (cache_type >= PERF_COUNT_HW_CACHE_MAX) return -EINVAL; cache_op = (config >> 8) & 0xff; if (cache_op >= PERF_COUNT_HW_CACHE_OP_MAX) return -EINVAL; cache_result = (config >> 16) & 0xff; if (cache_result >= PERF_COUNT_HW_CACHE_RESULT_MAX) return -EINVAL; code = (*tile_pmu->cache_events)[cache_type][cache_op][cache_result]; if (code == TILE_OP_UNSUPP) return -EINVAL; return code; } static void tile_event_destroy(struct perf_event *event) { if (atomic_dec_return(&tile_active_events) == 0) release_pmc_hardware(); } static int __tile_event_init(struct perf_event *event) { struct perf_event_attr *attr = &event->attr; struct hw_perf_event *hwc = &event->hw; int code; switch (attr->type) { case PERF_TYPE_HARDWARE: code = tile_pmu->map_hw_event(attr->config); break; case PERF_TYPE_HW_CACHE: code = tile_pmu->map_cache_event(attr->config); break; case PERF_TYPE_RAW: code = attr->config & TILE_EVENT_MASK; break; default: /* Should not happen. */ return -EOPNOTSUPP; } if (code < 0) return code; hwc->config = code; hwc->idx = -1; if (attr->exclude_user) hwc->config |= TILE_CTL_EXCL_USER; if (attr->exclude_kernel) hwc->config |= TILE_CTL_EXCL_KERNEL; if (attr->exclude_hv) hwc->config |= TILE_CTL_EXCL_HV; if (!hwc->sample_period) { hwc->sample_period = tile_pmu->max_period; hwc->last_period = hwc->sample_period; local64_set(&hwc->period_left, hwc->sample_period); } event->destroy = tile_event_destroy; return 0; } static int tile_event_init(struct perf_event *event) { int err = 0; perf_irq_t old_irq_handler = NULL; if (atomic_inc_return(&tile_active_events) == 1) old_irq_handler = reserve_pmc_hardware(tile_pmu_handle_irq); if (old_irq_handler) { pr_warn("PMC hardware busy (reserved by oprofile)\n"); atomic_dec(&tile_active_events); return -EBUSY; } switch (event->attr.type) { case PERF_TYPE_RAW: case PERF_TYPE_HARDWARE: case PERF_TYPE_HW_CACHE: break; default: return -ENOENT; } err = __tile_event_init(event); if (err) { if (event->destroy) event->destroy(event); } return err; } static struct pmu tilera_pmu = { .event_init = tile_event_init, .add = tile_pmu_add, .del = tile_pmu_del, .start = tile_pmu_start, .stop = tile_pmu_stop, .read = tile_pmu_read, }; /* * PMU's IRQ handler, PMU has 2 interrupts, they share the same handler. */ int tile_pmu_handle_irq(struct pt_regs *regs, int fault) { struct perf_sample_data data; struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); struct perf_event *event; struct hw_perf_event *hwc; u64 val; unsigned long status; int bit; __this_cpu_inc(perf_irqs); if (!atomic_read(&tile_active_events)) return 0; status = pmc_get_overflow(); pmc_ack_overflow(status); for_each_set_bit(bit, &status, tile_pmu->num_counters) { event = cpuc->events[bit]; if (!event) continue; if (!test_bit(bit, cpuc->active_mask)) continue; hwc = &event->hw; val = tile_perf_event_update(event); if (val & (1ULL << (tile_pmu->cntval_bits - 1))) continue; perf_sample_data_init(&data, 0, event->hw.last_period); if (!tile_event_set_period(event)) continue; if (perf_event_overflow(event, &data, regs)) tile_pmu_stop(event, 0); } return 0; } static bool __init supported_pmu(void) { tile_pmu = &tilepmu; return true; } int __init init_hw_perf_events(void) { supported_pmu(); perf_pmu_register(&tilera_pmu, "cpu", PERF_TYPE_RAW); return 0; } arch_initcall(init_hw_perf_events); /* Callchain handling code. */ /* * Tile specific backtracing code for perf_events. */ static inline void perf_callchain(struct perf_callchain_entry_ctx *entry, struct pt_regs *regs) { struct KBacktraceIterator kbt; unsigned int i; /* * Get the address just after the "jalr" instruction that * jumps to the handler for a syscall. When we find this * address in a backtrace, we silently ignore it, which gives * us a one-step backtrace connection from the sys_xxx() * function in the kernel to the xxx() function in libc. * Otherwise, we lose the ability to properly attribute time * from the libc calls to the kernel implementations, since * oprofile only considers PCs from backtraces a pair at a time. */ unsigned long handle_syscall_pc = handle_syscall_link_address(); KBacktraceIterator_init(&kbt, NULL, regs); kbt.profile = 1; /* * The sample for the pc is already recorded. Now we are adding the * address of the callsites on the stack. Our iterator starts * with the frame of the (already sampled) call site. If our * iterator contained a "return address" field, we could have just * used it and wouldn't have needed to skip the first * frame. That's in effect what the arm and x86 versions do. * Instead we peel off the first iteration to get the equivalent * behavior. */ if (KBacktraceIterator_end(&kbt)) return; KBacktraceIterator_next(&kbt); /* * Set stack depth to 16 for user and kernel space respectively, that * is, total 32 stack frames. */ for (i = 0; i < 16; ++i) { unsigned long pc; if (KBacktraceIterator_end(&kbt)) break; pc = kbt.it.pc; if (pc != handle_syscall_pc) perf_callchain_store(entry, pc); KBacktraceIterator_next(&kbt); } } void perf_callchain_user(struct perf_callchain_entry_ctx *entry, struct pt_regs *regs) { perf_callchain(entry, regs); } void perf_callchain_kernel(struct perf_callchain_entry_ctx *entry, struct pt_regs *regs) { perf_callchain(entry, regs); }