simd-team/work-ts
7
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

mix-object: remove

Browse Source
This commit is contained in:
Yingchi Long
2024-06-22 15:58:34 +08:00
parent 36e21ec7d6
commit 694154ee34
1 changed files with 0 additions and 543 deletions
Download Patch File Download Diff File Expand all files Collapse all files

543
src/mix-object.ts
View File

@@ -1,543 +0,0 @@
/**
* Binary search the first bad object in object sets.
* Things todo:
* 1. Determine the set of vector, or scalar version.
* 2. Use the mechanism above, construct such set
* 3. Link the set to an executable -> SPEC path, invoke "runcpu"
*/
import { autoBisect, binSearch } from "./algorithm";
import { pop2, SPEC, Bench, mkBench, mkSPEC, defaultSPEC, watchSPEC, parseSPECCSVResultsTable } from "./spec";
import path from "path";
import { FLANG, LLVM_EXTRACT, PREFIX, SW_AUTOVEC, SYSROOT_PREFIX, projectRoot } from "./environment";
import { extractCommand, generalCommand, linkerCommand } from "./commands/compiler";
import { runcpuOptions } from "./commands/spec";
import { checkedSpawnSync, promisifySpawn } from "./cli";
import { systemdRunOptions } from "./commands/systemd";
import { spawn } from "child_process";
import { functionList } from "./llvm-parser";
import assert from "assert";
import fs from "fs/promises";
import crypto from "crypto";
export interface Linkable {
link: (objectPath: (objectNames: string[]) => string[]) => Promise<void>;
}
function mkPop2Bench(spec: SPEC): Bench & Linkable {
const base = mkBench(spec, pop2);
return {
...base,
async link(objectPath) {
// Set up environment variables.
// FIXME: use -rpath or -rpath-link after I find out why.
const objects = objectPath(base.benchData().objectNames);
const options = [
...objects,
...linkerCommand({
searchDirs: [path.join(PREFIX, "lib"),],
}),
...generalCommand({
output: base.exepath(),
outputKind: "exe",
sysroot: SYSROOT_PREFIX,
}),
];
// Execute the link command
const proc = checkedSpawnSync(FLANG, options, {
stdio: "inherit",
env: {
...process.env,
LD_LIBRARY_PATH: [
path.join(PREFIX, 'lib'),
path.join(SYSROOT_PREFIX, 'lib'),
path.join(SYSROOT_PREFIX, 'usr', 'lib')
].join(':')
},
});
if (proc.error) {
throw proc.error;
}
},
};
}
// Bisect results.
const beforeHalfGrid = [
// 0.336
// Scalar [0, 161): Error,
// /home/lyc/workspace/sw-autovec/spec2017/result/CPU2017.671.fpspeed.refspeed.txt
// : /home/lyc/workspace/sw-autovec/spec2017/result/CPU2017.664.fpspeed.refspeed.txt
// Scalar [161, 322): Correct.
false,
// Scalar [161, 241): Error,
// /home/lyc/workspace/sw-autovec/spec2017/result/CPU2017.672.fpspeed.refspeed.txt
// Scalar [241, 322): Error,
// /home/lyc/workspace/sw-autovec/spec2017-2/result/CPU2017.672.fpspeed.refspeed.rsf
// Vector [161, 241): Error,
// Vector [241, 322): Error,
// /home/lyc/workspace/sw-autovec/spec2017-2/result/CPU2017.673.fpspeed.refspeed.txt
false,
// Vector [241, 281): Error,
// /home/lyc/workspace/sw-autovec/spec2017/result/CPU2017.674.fpspeed.refspeed.txt
// Vector [281, 322): Correct.
true,
// Vector [241, 261): Error,
// /home/lyc/workspace/sw-autovec/spec2017/result/CPU2017.675.fpspeed.refspeed.rsf
// Vector [261, 281): Correct.
true,
// Vector [241, 251): Error,
// /home/lyc/workspace/sw-autovec/spec2017/result/CPU2017.676.fpspeed.refspeed.txt
// Vector [251, 261): Stopped.
true,
// Vector [241, 246): Error,
// /home/lyc/workspace/sw-autovec/spec2017/result/CPU2017.677.fpspeed.refspeed.rsf
];
const beforeHalfNoGrid: boolean[] = [
false,
// Vector [161, 241): Error
// /home/lyc/workspace/sw-autovec/spec2017/result/CPU2017.679.
// Vector [241, 322): Unknown
true,
// Vector [161, 201): Unknown
// Vector [201, 241): Error
false,
// Vector [201, 221): Error
// /home/lyc/workspace/sw-autovec/spec2017/result/CPU2017.681
// Vector [221, 241): Unknown
true,
// Vector [201, 211): Error
// Vector [211, 221): Unknown
true,
// Vector [201, 206): Unknown
// Vector [206, 211): Error
false,
// Vector [206, 208): Error
// Vector [208, 211): ?
true,
// Vector [206, 207): ?
// !!! Vector [207, 208): Error
];
async function bisect(
beforeHalf: boolean[],
/** specify which spec to use */
rangeSPEC: (range: [number, number], index: number) => SPEC,
/** How to get the vector object path */
vector: (name: string) => string,
/** And how to get scalar path. */
scalar: (name: string) => string,
) {
const [vBegin, vHalf, vEnd] = binSearch(0, pop2.objectNames.length, beforeHalf);
// use scalar version if index is in this range.
const vectorRange: [number, number][] = [
[vBegin, vHalf],
[vHalf, vEnd],
];
await Promise.all(vectorRange.map((range, index) => ({
range,
// Assign each range a dedicated SPEC, to avoid race conditions.
spec: rangeSPEC(range, index),
})).map(async ({ range, spec }) => {
// For each range, linking the objects and test it.
const [a, b] = range;
const pop2Bench = mkPop2Bench(spec);
console.log(`Linking objects. Range in [${a}, ${b}) used vector version. others are scalar version.`);
await pop2Bench.link(names =>
names.map((name, index) => {
if (name === "grid.fppized.o" || index == 207) {
console.log(`${name} is a bad object!`);
return scalar(name);
}
// If index is in range [a, b), use the vector version.
return a <= index && index < b ? vector(name) : scalar(name);
}));
const serviceName = `${pop2Bench.benchData().num}-${a}-${b}`;
checkedSpawnSync("systemd-run",
[
...systemdRunOptions({
scope: false,
unit: `spec-${serviceName}`,
user: true,
}),
"runcpu",
...runcpuOptions({
benchmarks: [pop2Bench.benchData().num.toString()],
config: "clang-O2.cfg",
workload: "ref",
buildType: "nobuild",
outputFormat: ["text", "config"]
}),
], { stdio: "inherit", env: spec.getEnvironment() });
}));
};
const buildPathObj = (buildBase: string) => (build: string) => (objname: string) => path.join(buildBase, build, objname);
const buildDirs = ["build_vector", "build_scalar_vanilla"];
async function verifyGridFunction() {
const specs = [
"spec2017",
"spec2017-2",
].map(s => path.resolve(SW_AUTOVEC, s)).map(mkSPEC);
// To test which version works, for grid.
await (async (spec: SPEC) => {
const pop2Bench = mkPop2Bench(spec);
const [vector, scalar] = buildDirs.map(dir => buildPathObj(spec.buildpath(pop2))(dir));
await pop2Bench.link(names => {
return names.flatMap(p => {
if (p == "grid.fppized.o") {
const functionList = [
"grid_.o",
"grid_area_masks_.o",
"grid_calc_tpoints_.o",
"grid_cf_area_avg_.o",
"grid_compute_dz_.o",
"grid_fill_points_.o",
"grid_horiz_grid_internal_.o",
"grid_init_grid1_.o",
"grid_init_grid2_.o",
"grid_landmasks_.o",
"grid_read_bottom_cell_.o",
"grid_read_horiz_grid_.o",
"grid_read_topography_.o",
"grid_read_vert_grid_.o",
"grid_remove_isolated_points_.o",
"grid_remove_points_.o",
"grid_smooth_topography_.o",
"grid_tgrid_to_ugrid_.o",
"grid_topography_bathymetry_.o",
"grid_topography_internal_.o",
"grid_ugrid_to_tgrid_.o",
"grid_vert_grid_internal_.o",
];
return [
...functionList.map(fn => {
const getFunc = (name: string) => path.join(spec.buildpath(pop2), name, fn);
return fn == "grid_landmasks_.o" ? getFunc("function_scalar") : getFunc("function_simd");
}),
path.join(spec.buildpath(pop2), "global-obj.ll.o")
];
}
return vector(p);
});
});
await promisifySpawn(spawn("systemd-run",
[
...systemdRunOptions({
scope: true,
user: true,
unit: "pop2-grid-test",
}),
"runcpu",
...runcpuOptions({
benchmarks: [pop2Bench.benchData().num.toString()],
config: "clang-O2",
workload: "test",
buildType: "nobuild",
outputFormat: ["text", "config"]
}),
], { stdio: "inherit", env: spec.getEnvironment() }));
})(specs[0]);
};
async function checkBaro() {
// It is known that object index == 207 is also wrong for pop2.
const spec = defaultSPEC;
const troubleName = pop2.objectNames[207].replace(/\.o$/, "");
const pop2bench = mkPop2Bench(spec);
const local = path.resolve(projectRoot, "local");
// Maps from name of function -> path to object file.
const localSuffix = (suffix: string) => (fn: string) => path.resolve(local, `${fn}-${suffix}.o`);
const vectorObject = localSuffix("vector");
const scalarObject = localSuffix("scalar");
const build = path.resolve(spec.buildpath(pop2), "build_vector_test");
// Compile that object to "ll"
async function compileIR() {
const llvmIR = spawn(FLANG, ([
...generalCommand({
outputKind: "assembly",
output: "-",
sysroot: SYSROOT_PREFIX,
}),
"-emit-llvm",
"-O2",
"-msimd",
"-Mbyteswapio",
"-fno-discard-value-names",
path.resolve(build, `${troubleName}.f90`),
]), { stdio: ['inherit', 'pipe', 'inherit'], cwd: build });
const chunks: Buffer[] = [];
llvmIR.stdout.on("data", chunk => { chunks.push(Buffer.from(chunk)); });
const llvmIRResult = await promisifySpawn(llvmIR);
llvmIRResult.stdout;
return Buffer.concat(chunks).toString("utf-8");
}
const objFlags = [
...generalCommand({
outputKind: "object",
}),
"-x", "ir",
// Read from stdin.
"-",
];
const IR = await compileIR();
// First prepare a functions dir that would be suitable for replacing it's ".o"
async function prepareDir(IR: string, vectorObject: (fn: string) => string, scalarObject: (fn: string) => string) {
// Parse the IR with all function names
const fnList = functionList(IR);
// For each function, extract it.
await Promise.all(fnList.map(fn => {
const proc = spawn(LLVM_EXTRACT, [
...extractCommand({
func: [fn],
asm: true,
input: "-",
output: "-",
}),
]);
// Write IR and close stdin
proc.stdin.write(IR);
proc.stdin.end();
const compileVector = spawn(FLANG, [
...objFlags,
...generalCommand({ output: vectorObject(fn), }),
"-msimd",
]);
const compileScalar = spawn(FLANG, [
...objFlags,
...generalCommand({ output: scalarObject(fn) }),
]);
proc.stdout.pipe(compileVector.stdin);
proc.stdout.pipe(compileScalar.stdin);
return Promise.all([proc, compileScalar, compileScalar].map(promisifySpawn));
}));
return fnList;
}
const fnList = await prepareDir(IR, vectorObject, scalarObject);
// Also extract global names.
const globalObj = path.resolve(local, "207-global.o");
await (async () => {
const proc = spawn(LLVM_EXTRACT, [
...extractCommand({
asm: true,
input: "-",
output: "-",
}),
// Extract all global variables.
"-rglob",
".*",
], { stdio: ['pipe', 'pipe', 'inherit'] });
// Write IR and close stdin
proc.stdin.write(IR);
proc.stdin.end();
const compileScalar = spawn(FLANG, [
...objFlags,
...generalCommand({ output: globalObj }),
], { stdio: ['pipe', 'inherit', 'inherit'] });
proc.stdout.pipe(compileScalar.stdin);
return Promise.all([proc, compileScalar].map(promisifySpawn));
})();
// Bisect with replacing functions.
const [vector, scalar] = buildDirs.map(dir => buildPathObj(spec.buildpath(pop2))(dir));
// Run many SPEC2017, with different "outputRoot"
for (const fn of fnList) {
console.log(`testing function ${fn}`);
// For each function, try use a scalar version of that func
// Others are vector version.
const linkingList = fnList.map(name => name == fn ? scalarObject(name) : vectorObject(name));
// Link together with other objects.
await pop2bench.link(objectNames => objectNames.flatMap((objectName, index) => {
if (index == 207) {
return [...linkingList, globalObj];
}
if (objectName == "grid.fppized.o") {
return scalar(objectName);
}
return vector(objectName);
}));
const unitName = `pop2-${fn}-${crypto.randomUUID()}`;
// Spawn SPEC process.
const runcpu = spawn("systemd-run", [
...systemdRunOptions({
scope: true,
user: true,
unit: unitName,
}),
"runcpu",
...runcpuOptions({
benchmarks: [pop2.name],
buildType: "nobuild",
config: "clang-O2",
workload: "ref",
setprocgroup: true,
})
], { env: spec.getEnvironment() });
let OK = false;
setTimeout(() => {
if (!OK) {
console.log("timeout, kill the process.");
checkedSpawnSync(
"systemctl", [
"--user",
"stop",
`${unitName}.scope`,
], { stdio: "inherit" }
);
}
}, 10 * 60 * 1000); // 10min
runcpu.stdout.pipe(process.stdout);
runcpu.stderr.pipe(process.stderr);
const result = await watchSPEC(runcpu);
OK = true;
console.log(result);
if ("CSV" in result.outputFiles) {
assert(result.outputFiles.CSV.length > 0);
const Results = parseSPECCSVResultsTable((await fs.readFile(result.outputFiles.CSV[0])).toString("utf-8"));
let perlbenchResult;
if (perlbenchResult = Results[`${pop2.num}-${pop2.name}`]) {
if (perlbenchResult.baseStatus !== "S") {
console.log("Error!");
}
}
} else {
console.log("SPEC does not produce CSV output!");
}
}
};
async function bisectRef() {
const spec = defaultSPEC;
const [vector, scalar] = buildDirs.map(dir => buildPathObj(spec.buildpath(pop2))(dir));
const pop2Bench = mkPop2Bench(spec);
return await autoBisect(0, pop2.objectNames.length, async (begin, end) => {
console.log(`Checking range [${begin}, ${end})`);
console.log("Linking pop2 executable");
await pop2Bench.link(names =>
names.map((name, index) => {
// These two objects are known to have miscompilation.
if (name === "grid.fppized.o" || index == 207) {
return scalar(name);
}
// If index is in range [begin, end), use the vector version.
return begin <= index && index < end ? vector(name) : scalar(name);
}));
console.log("Running SPEC2017 benchmark.");
const specProcess = spawn("runcpu",
[
...runcpuOptions({
benchmarks: [pop2Bench.benchData().num.toString()],
config: "clang-O2.cfg",
workload: "ref",
buildType: "nobuild",
outputFormat: ["csv", "config"]
}),
], { env: spec.getEnvironment() });
const result = await watchSPEC(specProcess);
if ("CSV" in result.outputFiles) {
assert(result.outputFiles.CSV.length > 0);
const Results = parseSPECCSVResultsTable((await fs.readFile(result.outputFiles.CSV[0]))
.toString("utf-8"));
const benchResult = Results[`${pop2.num}.${pop2.name}`];
if (benchResult) {
const status = benchResult.baseStatus;
console.log(`status of this run: ${status}`);
return status !== "S";
}
assert(false, "Result should have pop2 field!");
} else {
console.log("SPEC does not produce CSV output!");
return true;
}
});
}
(async () => {
console.log(await bisectRef());
})();