simd-team/work-ts
7
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

spec: refactor to a dedicated directory

Browse Source
This commit is contained in:
Yingchi Long
2024-06-23 20:14:34 +08:00
parent 505c5ef752
commit 0e29cad5ec
3 changed files with 205 additions and 197 deletions
Download Patch File Download Diff File Expand all files Collapse all files

25
src/spec/benchData.ts Normal file
View File

@@ -0,0 +1,25 @@
import path from 'path';
export interface HaveSPECObjects {
objectNames: string[];
}
export interface SPECBenchData {
num: number;
name: string;
exe: string;
}
export function benchpath(specdir: string, bench: SPECBenchData): string {
return path.join(specdir, 'benchspec', 'CPU', `${bench.num}.${bench.name}`);
}
export function exepath(specdir: string, bench: SPECBenchData): string {
const benchmarkDir = benchpath(specdir, bench);
return path.join(benchmarkDir, 'exe', bench.exe);
}
export function buildpath(specdir: string, bench: SPECBenchData): string {
const benchmarkDir = benchpath(specdir, bench);
return path.join(benchmarkDir, 'build');
}

178
src/spec/index.ts Normal file
View File

@@ -0,0 +1,178 @@
import * as path from 'path';
import * as fs from "fs";
import { promisify } from "util";
import { ChildProcessByStdio } from 'child_process';
import { Readable, Writable } from 'stream';
import { createInterface } from 'readline';
import { projectRoot } from 'lyc/environment';
import { SPECBenchData, benchpath, buildpath, exepath } from './benchData';
export{ pop2 } from './pop2';
export function getEnvironment(specdir: string): NodeJS.ProcessEnv {
return {
...process.env,
SPEC: specdir,
PATH: `${path.join(specdir, 'bin')}${path.delimiter}${process.env.PATH}`,
};
}
export interface SPEC {
newConfig: (name: string, content: string) => Promise<void>;
benchpath: (bench: SPECBenchData) => string;
exepath: (bench: SPECBenchData) => string;
buildpath: (bench: SPECBenchData) => string;
getEnvironment: () => NodeJS.ProcessEnv;
}
export function mkSPEC(specRoot: string): SPEC {
return {
newConfig: async (name: string, content: string) => {
await writeFile(path.join(specRoot, "config", name), content);
},
benchpath: bench => benchpath(specRoot, bench),
exepath: bench => exepath(specRoot, bench),
buildpath: bench => buildpath(specRoot, bench),
getEnvironment: () => getEnvironment(specRoot),
};
}
export interface Bench {
benchpath: () => string;
exepath: () => string;
buildpath: () => string;
benchData: () => SPECBenchData;
spec: () => SPEC;
}
export function mkBench(spec: SPEC, bench: SPECBenchData): Bench {
return {
benchpath: () => spec.benchpath(bench),
exepath: () => spec.exepath(bench),
buildpath: () => spec.buildpath(bench),
benchData: () => bench,
spec: () => spec,
};
}
const specTemplate = fs.readFileSync(path.resolve(projectRoot, "assets", "specTemplate.cfg")).toString("utf-8");
const writeFile = promisify(fs.writeFile);
export interface ConfigOptions {
gccdir: string;
optimize: string;
}
export function renderConfig(options: ConfigOptions) {
return `# Rendered from TypeScript ${new Date().toLocaleString()}, do not edit!\n\n\n` + specTemplate
.replace("@@GCCDIR@@", options.gccdir)
.replace("@@OPTIMIZE@@", options.optimize);
}
export interface SPECResult {
outputFiles: { [key: string]: string[], };
}
/**
* Monitor SPEC2017 runcpu process in a subprocess.
*/
export async function watchSPEC<T extends ChildProcessByStdio<null | Writable, Readable, Readable>>(process: T) {
const rl = createInterface({
input: process.stdout,
terminal: false,
});
// Match & extract string like " format CSV -> /path/to/csv"
const formatRe = /^\s*format:\s*(\w+)\s*->\s*(.*)$/;
return await new Promise<SPECResult>((resolve, reject) => {
let outputFiles: { [key: string]: string[]; } = {};
rl.on("line", line => {
let match;
if ((match = formatRe.exec(line)) != null) {
const type = match[1];
const paths = match[2].split(',').map(path => path.trim());
outputFiles[type] = paths;
}
// To match if the line contains: "runcpu finished
if (line.startsWith("runcpu finished")) {
resolve({
outputFiles
});
}
});
process.on("error", error => reject(error));
process.on("close", () => resolve({
outputFiles
}));
});
}
export interface SPECResultsTable {
baseThreads: string,
baseRuntime: string,
baseRatio: string,
BaseSelected: string,
baseStatus: string,
peakThreads: string,
peakRuntime: string,
peakRatio: string,
peakSelected: string,
peakStatus: string,
description?: string,
}
export function parseSPECCSVResultsTable(data: string) {
const lines = data.split("\n");
let index = 0;
const eof = () => index >= lines.length;
let results: { [key: string]: SPECResultsTable; } = {};
while (!eof()) {
const line = lines[index];
if (line.startsWith('"Full Results Table"')) {
index += 3;
// "Full Results Table"
// Benchmark,"Base # Threads","Est. Base Run Time","Est. Base Ratio","Base Selected","Base Status","Peak # Threads","Est. Peak Run Time","Est. Peak Ratio","Peak Selected","Peak Status",Description
// 600.perlbench_s,1,65.357247,--,0,VE,,,,,,"test iteration #1"
// 602.gcc_s,,,,,NR,,,,,NR
// 605.mcf_s,,,,,NR,,,,,NR
// 620.omnetpp_s,,,,,NR,,,,,NR
// 623.xalancbmk_s,,,,,NR,,,,,NR
// 625.x264_s,,,,,NR,,,,,NR
// 631.deepsjeng_s,,,,,NR,,,,,NR
// 641.leela_s,,,,,NR,,,,,NR
// 648.exchange2_s,,,,,NR,,,,,NR
// 657.xz_s,,,,,NR,,,,,NR
while (!eof()) {
const dataLine = lines[index];
if (dataLine.length == 0)
break;
const fields: (keyof SPECResultsTable)[] = [
"baseThreads",
"baseRuntime",
"baseRatio",
"BaseSelected",
"baseStatus",
"peakThreads",
"peakRuntime",
"peakRatio",
"peakSelected",
"peakStatus",
"description",
];
const splitted = dataLine.split(",");
const entries: [keyof SPECResultsTable, string][] = splitted.slice(1).map((name, index) => [fields[index], name]);
results[splitted[0]] = (Object.fromEntries(entries) as { [K in keyof SPECResultsTable]: string });
index++;
}
}
index++;
}
return results;
}

331
src/spec/pop2.ts Normal file
View File

@@ -0,0 +1,331 @@
import { HaveSPECObjects, SPECBenchData } from "./benchData";
export const pop2: SPECBenchData & HaveSPECObjects = {
objectNames: [
"netcdf/attr.o",
"netcdf/dim.o",
"netcdf/error.o",
"netcdf/fort-attio.o",
"netcdf/fort-control.o",
"netcdf/fort-dim.o",
"netcdf/fort-genatt.o",
"netcdf/fort-geninq.o",
"netcdf/fort-genvar.o",
"netcdf/fort-lib.o",
"netcdf/fort-misc.o",
"netcdf/fort-v2compat.o",
"netcdf/fort-var1io.o",
"netcdf/fort-varaio.o",
"netcdf/fort-vario.o",
"netcdf/fort-varmio.o",
"netcdf/fort-varsio.o",
"netcdf/libvers.o",
"netcdf/nc.o",
"netcdf/ncx.o",
"netcdf/posixio.o",
"netcdf/putget.o",
"netcdf/string.o",
"netcdf/v1hpg.o",
"netcdf/v2i.o",
"netcdf/var.o",
"netcdf/netcdf.o",
"netcdf/typeSizes.o",
"fort.fppized.o",
"mpi.o",
"send.o",
"recv.o",
"collective.o",
"req.o",
"list.o",
"handles.o",
"comm.o",
"group.o",
"time.o",
"pack.o",
"m_IndexBin_char.fppized.o",
"m_IndexBin_integer.fppized.o",
"m_IndexBin_logical.fppized.o",
"m_List.fppized.o",
"m_MergeSorts.fppized.o",
"m_Filename.fppized.o",
"m_FcComms.fppized.o",
"m_Permuter.fppized.o",
"m_SortingTools.fppized.o",
"m_String.fppized.o",
"m_StrTemplate.fppized.o",
"m_chars.fppized.o",
"m_die.fppized.o",
"m_dropdead.fppized.o",
"m_FileResolv.fppized.o",
"m_flow.fppized.o",
"m_inpak90.fppized.o",
"m_ioutil.fppized.o",
"m_mall.fppized.o",
"m_mpif.fppized.o",
"m_mpif90.fppized.o",
"m_mpout.fppized.o",
"m_rankMerge.fppized.o",
"m_realkinds.fppized.o",
"m_stdio.fppized.o",
"m_TraceBack.fppized.o",
"m_zeit.fppized.o",
"get_zeits.o",
"m_MCTWorld.fppized.o",
"m_AttrVect.fppized.o",
"m_GlobalMap.fppized.o",
"m_GlobalSegMap.fppized.o",
"m_GlobalSegMapComms.fppized.o",
"m_Accumulator.fppized.o",
"m_SparseMatrix.fppized.o",
"m_Navigator.fppized.o",
"m_AttrVectComms.fppized.o",
"m_AttrVectReduce.fppized.o",
"m_AccumulatorComms.fppized.o",
"m_GeneralGrid.fppized.o",
"m_GeneralGridComms.fppized.o",
"m_SpatialIntegral.fppized.o",
"m_SpatialIntegralV.fppized.o",
"m_MatAttrVectMul.fppized.o",
"m_Merge.fppized.o",
"m_GlobalToLocal.fppized.o",
"m_ExchangeMaps.fppized.o",
"m_ConvertMaps.fppized.o",
"m_SparseMatrixDecomp.fppized.o",
"m_SparseMatrixToMaps.fppized.o",
"m_SparseMatrixComms.fppized.o",
"m_SparseMatrixPlus.fppized.o",
"m_Router.fppized.o",
"m_Rearranger.fppized.o",
"m_Transfer.fppized.o",
"alloc_mod.fppized.o",
"box_rearrange.fppized.o",
"iompi_mod.fppized.o",
"ionf_mod.fppized.o",
"mct_rearrange.fppized.o",
"nf_mod.fppized.o",
"piodarray.fppized.o",
"pio.fppized.o",
"pio_kinds.fppized.o",
"piolib_mod.fppized.o",
"pio_mpi_utils.fppized.o",
"pionfatt_mod.fppized.o",
"pionfget_mod.fppized.o",
"pionfput_mod.fppized.o",
"pionfread_mod.fppized.o",
"pio_nf_utils.fppized.o",
"pionfwrite_mod.fppized.o",
"pio_quicksort.fppized.o",
"pio_spmd_utils.fppized.o",
"pio_support.fppized.o",
"pio_types.fppized.o",
"pio_utils.fppized.o",
"pnetcdfversion.o",
"rearrange.fppized.o",
"topology.o",
"dead_data_mod.fppized.o",
"dead_mct_mod.fppized.o",
"dead_mod.fppized.o",
"ESMF_AlarmClockMod.fppized.o",
"ESMF_AlarmMod.fppized.o",
"ESMF_BaseMod.fppized.o",
"ESMF_BaseTimeMod.fppized.o",
"ESMF_CalendarMod.fppized.o",
"ESMF_ClockMod.fppized.o",
"ESMF_FractionMod.fppized.o",
"ESMF_Mod.fppized.o",
"ESMF_Stubs.fppized.o",
"ESMF_TimeIntervalMod.fppized.o",
"ESMF_TimeMod.fppized.o",
"f_wrappers.o",
"GPTLget_memusage.o",
"GPTLprint_memusage.o",
"GPTLutil.o",
"mct_mod.fppized.o",
"Meat.fppized.o",
"perf_mod.fppized.o",
"perf_utils.fppized.o",
"seq_cdata_mod.fppized.o",
"seq_comm_mct.fppized.o",
"seq_drydep_mod.fppized.o",
"seq_flds_indices.fppized.o",
"seq_flds_mod.fppized.o",
"seq_infodata_mod.fppized.o",
"seq_io_mod.fppized.o",
"seq_timemgr_mod.fppized.o",
"shr_cal_mod.fppized.o",
"shr_const_mod.fppized.o",
"shr_dmodel_mod.fppized.o",
"shr_file_mod.fppized.o",
"shr_flux_mod.fppized.o",
"shr_jlcp.o",
"shr_kind_mod.fppized.o",
"shr_log_mod.fppized.o",
"shr_map_mod.fppized.o",
"shr_mct_mod.fppized.o",
"shr_mem_mod.fppized.o",
"shr_mpi_mod.fppized.o",
"shr_msg_mod.fppized.o",
"shr_ncread_mod.fppized.o",
"shr_orb_mod.fppized.o",
"shr_pcdf_mod.fppized.o",
"shr_scam_mod.fppized.o",
"shr_strdata_mod.fppized.o",
"shr_stream_mod.fppized.o",
"shr_string_mod.fppized.o",
"shr_sys_mod.fppized.o",
"shr_timer_mod.fppized.o",
"shr_tInterp_mod.fppized.o",
"shr_vmath_fwrap.o",
"shr_vmath_mod.fppized.o",
"threadutil.o",
"wrf_error_fatal.fppized.o",
"wrf_message.fppized.o",
"atm_comp_mct.fppized.o",
"datm_comp_mod.fppized.o",
"datm_shr_mod.fppized.o",
"dlnd_comp_mod.fppized.o",
"lnd_comp_mct.fppized.o",
"dice_comp_mod.fppized.o",
"ice_comp_mct.fppized.o",
"POP_BlocksMod.fppized.o",
"POP_BroadcastMod.fppized.o",
"POP_CommMod.fppized.o",
"POP_ConfigMod.fppized.o",
"POP_ConstantsMod.fppized.o",
"POP_DistributionMod.fppized.o",
"POP_DomainSizeMod.fppized.o",
"POP_ErrorMod.fppized.o",
"POP_FieldMod.fppized.o",
"POP_FinalMod.fppized.o",
"POP_GridDimMod.fppized.o",
"POP_GridHorzMod.fppized.o",
"POP_GridVertMod.fppized.o",
"POP_HaloMod.fppized.o",
"POP_IOUnitsMod.fppized.o",
"POP_InitMod.fppized.o",
"POP_KindsMod.fppized.o",
"POP_MCT_vars_mod.fppized.o",
"POP_RedistributeMod.fppized.o",
"POP_ReductionsMod.fppized.o",
"POP_SolversMod.fppized.o",
"advection.fppized.o",
"baroclinic.fppized.o",
"barotropic.fppized.o",
"blocks.fppized.o",
"broadcast.fppized.o",
"budget_diagnostics.fppized.o",
"cfc11_mod.fppized.o",
"cfc_mod.fppized.o",
"check_mod.fppized.o",
"co2calc.fppized.o",
"communicate.fppized.o",
"constants.fppized.o",
"current_meters.fppized.o",
"diag_bsf.fppized.o",
"diagnostics.fppized.o",
"diags_on_lat_aux_grid.fppized.o",
"distribution.fppized.o",
"domain.fppized.o",
"domain_size.fppized.o",
"drifters.fppized.o",
"ecosys_mod.fppized.o",
"ecosys_parms.fppized.o",
"exit_mod.fppized.o",
"forcing.fppized.o",
"forcing_ap.fppized.o",
"forcing_coupled.fppized.o",
"forcing_fields.fppized.o",
"forcing_pt_interior.fppized.o",
"forcing_s_interior.fppized.o",
"forcing_sfwf.fppized.o",
"forcing_shf.fppized.o",
"forcing_tools.fppized.o",
"forcing_ws.fppized.o",
"gather_scatter.fppized.o",
"global_reductions.fppized.o",
"grid.fppized.o",
"history.fppized.o",
"hmix_aniso.fppized.o",
"hmix_del2.fppized.o",
"hmix_del4.fppized.o",
"hmix_gm.fppized.o",
"hmix_gm_submeso_share.fppized.o",
"horizontal_mix.fppized.o",
"hydro_sections.fppized.o",
"iage_mod.fppized.o",
"ice.fppized.o",
"initial.fppized.o",
"io.fppized.o",
"io_binary.fppized.o",
"io_ccsm.fppized.o",
"io_netcdf.fppized.o",
"io_pio.fppized.o",
"io_tools.fppized.o",
"io_types.fppized.o",
"kinds_mod.fppized.o",
"mix_submeso.fppized.o",
"movie.fppized.o",
"ms_balance.fppized.o",
"msg_mod.fppized.o",
"named_field_mod.fppized.o",
"ocn_communicator.fppized.o",
"ocn_comp_mct.fppized.o",
"operators.fppized.o",
"output.fppized.o",
"overflows.fppized.o",
"passive_tracer_tools.fppized.o",
"passive_tracers.fppized.o",
"pressure_grad.fppized.o",
"prognostic.fppized.o",
"qflux_mod.fppized.o",
"registry.fppized.o",
"restart.fppized.o",
"spacecurve_mod.fppized.o",
"state_mod.fppized.o",
"step_mod.fppized.o",
"surface_hgt.fppized.o",
"sw_absorption.fppized.o",
"tavg.fppized.o",
"tidal_mixing.fppized.o",
"time_management.fppized.o",
"timers.fppized.o",
"topostress.fppized.o",
"tracer_types.fppized.o",
"vertical_mix.fppized.o",
"vmix_const.fppized.o",
"vmix_kpp.fppized.o",
"vmix_rich.fppized.o",
"glc_comp_mct.fppized.o",
"ccsm_driver.fppized.o",
"map_atmatm_mct.fppized.o",
"map_atmice_mct.fppized.o",
"map_atmlnd_mct.fppized.o",
"map_atmocn_mct.fppized.o",
"map_glcglc_mct.fppized.o",
"map_iceice_mct.fppized.o",
"map_iceocn_mct.fppized.o",
"map_lndlnd_mct.fppized.o",
"map_ocnocn_mct.fppized.o",
"map_rofocn_mct.fppized.o",
"map_rofrof_mct.fppized.o",
"map_snoglc_mct.fppized.o",
"map_snosno_mct.fppized.o",
"mrg_x2a_mct.fppized.o",
"mrg_x2g_mct.fppized.o",
"mrg_x2i_mct.fppized.o",
"mrg_x2l_mct.fppized.o",
"mrg_x2o_mct.fppized.o",
"mrg_x2s_mct.fppized.o",
"seq_avdata_mod.fppized.o",
"seq_diag_mct.fppized.o",
"seq_domain_mct.fppized.o",
"seq_flux_mct.fppized.o",
"seq_frac_mct.fppized.o",
"seq_hist_mod.fppized.o",
"seq_rearr_mod.fppized.o",
"seq_rest_mod.fppized.o",
],
exe: "speed_pop2_base.mytest-m64",
name: "pop2_s",
num: 628,
};