diff --git a/.bin/srclib-javascript b/.bin/srclib-javascript new file mode 100755 index 0000000..592ef0d --- /dev/null +++ b/.bin/srclib-javascript @@ -0,0 +1,230 @@ +#!/usr/bin/env node + +var parser = require("nomnom"); +var findpkgs = require("commonjs-findpkgs"); +var jsgAdapter = require("../lib/jsg_adapter") +var npm = require("npm"); +var path = require("path"); +var execFile = require("child_process").execFile; + +parser.command("scan-commonjs") + .option("repo", { + help: "repository URI" + }) + .option("subdir", { + help: "subdirectory in repository" + }) + .callback(function(opts) { + // TODO(sqs): support ScanIgnore from repo Srcfile (pass to findpkgs ignore param) + findpkgs(".", ["node_modules"], function(err, pkgs) { + if (err) { + console.error("Scanning failed:", err); + process.exit(1); + } + + // filter out undesirable packages (packages whose paths contain + // "node_modules", "test", etc.) + pkgs = pkgs.filter(function(pkg) { + // TODO(sqs): we probably want to process things in node_modules for + // local code, so remove this filter. + var pathComps = pkg.dir.split("/"); + return pathComps.indexOf("node_modules") == -1 && pathComps.indexOf("test") == -1 && pathComps.indexOf("templates") == -1; + }); + + // filter out undesirable source files (minified files) from + // packages + pkgs.forEach(function(pkg) { + pkg.libFiles = pkg.libFiles.filter(function(f) { + return !/\.min\.js$/.test(f); + }); + }); + + // convert from commonjs-findpkgs format to source unit + var srcunits = pkgs.map(function(pkg) { + // collect all deps + var allDeps = []; + function collectDeps(deps) { + if (deps) Object.keys(deps).forEach(function(name) { allDeps.push({name: name, version: deps[name]}); }); + } + collectDeps(pkg.package.dependencies); + collectDeps(pkg.package.devDependencies); + + return { + Name: pkg.package.name, + Type: "CommonJSPackage", + Dir: pkg.dir, + Files: pkg.libFiles.concat(pkg.testFiles), + Dependencies: allDeps, + Data: pkg.package, + Ops: {depresolve: null, graph: null}, + Config: { + npmInstall: !!process.env.IN_DOCKER_CONTAINER, + jsg: { + plugins: { + node: { + coreModulesDir: process.env.NODEJS_CORE_MODULES_DIR || "$(JSG_DIR)/testdata/node_core_modules", + } + }, + }, + }, + }; + }); + + console.log(JSON.stringify(srcunits, null, 2)); + }); + }) + .help("scan for CommonJS packages"); + +parser.command("depresolve") + .callback(function(opts) { + var stdin = process.stdin, + stdout = process.stdout, + inputChunks = []; + + stdin.resume(); + stdin.setEncoding("utf8"); + + stdin.on("data", function (chunk) { + inputChunks.push(chunk); + }); + + stdin.on("end", function () { + var inputJSON = inputChunks.join(""); + var commonJSPackage = JSON.parse(inputJSON); + + var remaining = commonJSPackage.Dependencies ? commonJSPackage.Dependencies.length : 0; + var resolutions = []; + + if (remaining == 0) { + console.log('[]'); + return; + } + + commonJSPackage.Dependencies.forEach(function(dep) { + var i = commonJSPackage.Dependencies.indexOf(dep); + resolutions[i] = {Raw: dep}; + var spec = dep.name + "@" + (dep.version || 'latest'); + npm.load(function(err, npm) { + if (err) { + console.error("npm.load failed: ", err); + process.exit(1) + } + + npm.commands.view(["--silent", spec], function(err,data) { + if (err) { + resolutions[i].Error = "error occurred while resolving: " + JSON.stringify(err); + } else if (!data || Object.keys(data).length == 0) { + resolutions[i].Error = "no npm package found with spec " + JSON.stringify(spec); + } else { + //Choose most recent version + var info = Object.keys(data).slice(-1)[0]; + if (!info) { + var repoURL = ""; + if (info.repository) { + if (typeof info.repository == "string") repoURL = info.repository; + else repoURL = info.repository.url; + } + + resolutions[i].Target = { + ToRepoCloneURL: repoURL, + ToUnit: info.name, + ToUnitType: "CommonJSPackage", + ToVersionString: info.version + }; + } + } + + remaining--; + if (remaining == 0) { + // done resolving all + console.log(JSON.stringify(resolutions, null, 2)); + } + }); + }); + }); + }); + }) + .help("resolves dependencies of a CommonJS package"); + +parser.command("graph") + .callback(function(opts) { + var stdin = process.stdin, + stdout = process.stdout, + inputChunks = []; + + stdin.resume(); + stdin.setEncoding("utf8"); + + stdin.on("data", function (chunk) { + inputChunks.push(chunk); + }); + + stdin.on("end", function () { + var inputJSON = inputChunks.join(""); + var commonJSPackage = JSON.parse(inputJSON); + + function graph() { + jsgAdapter.run(".", commonJSPackage, function(err, graphData) { + if (err) { + console.error("Graphing " + commonJSPackage.Name + " failed:", err); + process.exit(1); + } + console.log(JSON.stringify(graphData, null, 2)); + }); + } + + // TODO(sqs) install npm deps if we're in a docker container, since we can't rely on + // them to have been installed locally. we need to account for the fact + // that we've mounted the source volume readonly, so we can't just run + // `npm install`. + if (process.env.IN_DOCKER_CONTAINER && commonJSPackage.Config.npmInstall) { + // running `npm install` in the /src dir fails because it's mounted + // readonly. but `npm install -g` in the /src dir does nothing. so we + // need to `npm install /src` from a dir other than /src to get `npm + // install` to do anything. + + + // TODO(sqs): only works for top-level package.jsons. we actually need + // "/src/${dirname(package.json path)", which also requires us to + // track the path to the package.json. + execFile("npm", ["install", "--ignore-scripts", "/src"], {cwd: "/"}, function(err, stdout, stderr) { + if (stderr) console.error(stderr); + if (stdout) console.error(stdout); + if (err) { + console.error("npm install in Docker container failed: ", err); + process.exit(1) + } + graph(); + }); + + + // It's apparently impossible to silence npm install output when using + // npm.install. :( + // var oldcwd = process.cwd(); + // process.chdir(path.join(oldcwd, "..")); + // console.error("npm installing packages in Docker container..."); + // npm.load({loglevel: "silent"}, function(err, npm) { + // if (err) { + // console.error("npm.load in Docker container failed: ", err); + // process.exit(1) + // } + // // TODO(sqs): only works for top-level package.jsons. we actually need + // // "/src/${dirname(package.json path)", which also requires us to + // // track the path to the package.json. + // npm.commands.install(["/src"], function(err,data) { + // if (err) { + // console.error("npm install in Docker container failed: ", err); + // process.exit(1) + // } + // process.chdir(oldcwd); + // graph(); + // }); + // }); + } else { + graph(); + } + }); + }) + .help("graph a CommonJS package using jsg/tern"); + +parser.parse(); diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..f34c6e1 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,5 @@ +.git +testdata +__old +.srclib-cache + diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..bffe043 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +testdata/actual/ diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..16280f5 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,9 @@ +[submodule "testdata/case/javascript-nodejs-sample-0"] + path = testdata/case/javascript-nodejs-sample-0 + url = https://github.com/sgtest/javascript-nodejs-sample-0.git +[submodule "testdata/case/javascript-nodejs-xrefs-0"] + path = testdata/case/javascript-nodejs-xrefs-0 + url = https://github.com/sgtest/javascript-nodejs-xrefs-0.git +[submodule "testdata/case/minimal_nodejs_stdlib"] + path = testdata/case/minimal_nodejs_stdlib + url = https://github.com/sgtest/minimal_nodejs_stdlib.git diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..92d7b19 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,12 @@ +language: node_js + +node_js: + - "0.11" + - "0.10" + +install: + - sudo wget -NO /tmp/src.zip 'https://api.equinox.io/1/Applications/ap_BQxVz1iWMxmjQnbVGd85V58qz6/Updates/Asset/src-0.0.22.zip?os=linux&arch=amd64&channel=stable' + - sudo unzip /tmp/src*.zip -d /usr/local/bin + - sudo mv /usr/local/bin/src-* /usr/local/bin/src + - sudo chmod +x /usr/local/bin/src + - src toolchain add sourcegraph.com/sourcegraph/srclib-javascript diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..5bdc30d --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,71 @@ +# How to Contribute + +Sourcegraph projects are [BSD licensed](LICENSE) and accept contributions via +GitHub pull requests. This document outlines some of the conventions on +development workflow, commit message formatting, contact points and other +resources to make it easier to get your contribution accepted. + +# Certificate of Origin + +By contributing to this project you agree to the Developer Certificate of Origin +(DCO). This document was created by the Linux Kernel community and is a simple +statement that you, as a contributor, have the legal right to make the +contribution. See the [DCO](DCO) file for details. + +# Email and Chat + +The project currently uses the general Sourcegraph email list and IRC channel: +- Email: [srclib-dev](https://groups.google.com/forum/#!forum/srclib-dev) +- IRC: #[srclib](irc://irc.freenode.org:6667/#srclib) IRC channel on freenode.org + +Please avoid emailing maintainers found in the MAINTAINERS file directly. They +are very busy and read the mailing lists. + +## Getting Started + +- Fork the repository on GitHub +- Read the [README](README.md) for build and test instructions +- Play with the project, submit bugs, submit patches! + +## Contribution Flow + +This is a rough outline of what a contributor's workflow looks like: + +- Create a topic branch from where you want to base your work (usually master). +- Make commits of logical units. +- Make sure your commit messages are in the proper format (see below). +- Push your changes to a topic branch in your fork of the repository. +- Make sure the tests pass, and add any new tests as appropriate. +- Submit a pull request to the original repository. + +Thanks for your contributions! + +### Format of the Commit Message + +We follow a rough convention for commit messages that is designed to answer two +questions: what changed and why. The subject line should feature the what and +the body of the commit should describe the why. + +``` +scripts: add the test-cluster command + +this uses tmux to setup a test cluster that you can easily kill and +start for debugging. + +Fixes #38 +``` + +The format can be described more formally as follows: + +``` +: + + + +