sourcegraph · xizhao · Jan 14, 2014 · Jan 14, 2014 · Jan 14, 2014 · Jan 15, 2014
diff --git a/.bin/srclib-javascript b/.bin/srclib-javascript
@@ -0,0 +1,230 @@
+#!/usr/bin/env node
+
+var parser = require("nomnom");
+var findpkgs = require("commonjs-findpkgs");
+var jsgAdapter = require("../lib/jsg_adapter")
+var npm = require("npm");
+var path = require("path");
+var execFile = require("child_process").execFile;
+
+parser.command("scan-commonjs")
+  .option("repo", {
+    help: "repository URI"
+  })
+  .option("subdir", {
+    help: "subdirectory in repository"
+  })
+  .callback(function(opts) {
+    // TODO(sqs): support ScanIgnore from repo Srcfile (pass to findpkgs ignore param)
+    findpkgs(".", ["node_modules"], function(err, pkgs) {
+      if (err) {
+        console.error("Scanning failed:", err);
+        process.exit(1);
+      }
+
+      // filter out undesirable packages (packages whose paths contain
+      // "node_modules", "test", etc.)
+      pkgs = pkgs.filter(function(pkg) {
+        // TODO(sqs): we probably want to process things in node_modules for
+        // local code, so remove this filter.
+        var pathComps = pkg.dir.split("/");
+        return pathComps.indexOf("node_modules") == -1 && pathComps.indexOf("test") == -1 && pathComps.indexOf("templates") == -1;
+      });
+
+      // filter out undesirable source files (minified files) from
+      // packages
+      pkgs.forEach(function(pkg) {
+        pkg.libFiles = pkg.libFiles.filter(function(f) {
+          return !/\.min\.js$/.test(f);
+        });
+      });
+
+      // convert from commonjs-findpkgs format to source unit
+      var srcunits = pkgs.map(function(pkg) {
+        // collect all deps
+        var allDeps = [];
+        function collectDeps(deps) {
+          if (deps) Object.keys(deps).forEach(function(name) { allDeps.push({name: name, version: deps[name]}); });
+        }
+        collectDeps(pkg.package.dependencies);
+        collectDeps(pkg.package.devDependencies);
+
+        return {
+          Name: pkg.package.name,
+          Type: "CommonJSPackage",
+          Dir: pkg.dir,
+          Files: pkg.libFiles.concat(pkg.testFiles),
+          Dependencies: allDeps,
+          Data: pkg.package,
+          Ops: {depresolve: null, graph: null},
+          Config: {
+            npmInstall: !!process.env.IN_DOCKER_CONTAINER,
+            jsg: {
+              plugins: {
+                node: {
+                  coreModulesDir: process.env.NODEJS_CORE_MODULES_DIR || "$(JSG_DIR)/testdata/node_core_modules",
+                }
+              },
+            },
+          },
+        };
+      });
+
+      console.log(JSON.stringify(srcunits, null, 2));
+    });
+  })
+  .help("scan for CommonJS packages");
+
+parser.command("depresolve")
+  .callback(function(opts) {
+    var stdin = process.stdin,
+    stdout = process.stdout,
+    inputChunks = [];
+
+    stdin.resume();
+    stdin.setEncoding("utf8");
+
+    stdin.on("data", function (chunk) {
+      inputChunks.push(chunk);
+    });
+
+    stdin.on("end", function () {
+      var inputJSON = inputChunks.join("");
+      var commonJSPackage = JSON.parse(inputJSON);
+
+      var remaining = commonJSPackage.Dependencies ? commonJSPackage.Dependencies.length : 0;
+      var resolutions = [];
+
+      if (remaining == 0) {
+        console.log('[]');
+        return;
+      }
+
+      commonJSPackage.Dependencies.forEach(function(dep) {
+        var i = commonJSPackage.Dependencies.indexOf(dep);
+        resolutions[i] = {Raw: dep};
+        var spec = dep.name + "@" + (dep.version || 'latest');
+        npm.load(function(err, npm) {
+          if (err) {
+            console.error("npm.load failed: ", err);
+            process.exit(1)
+          }
+
+          npm.commands.view(["--silent", spec], function(err,data) {
+            if (err) {
+              resolutions[i].Error = "error occurred while resolving: " + JSON.stringify(err);
+            } else if (!data || Object.keys(data).length == 0) {
+              resolutions[i].Error = "no npm package found with spec " + JSON.stringify(spec);
+            } else {
+              //Choose most recent version
+              var info = Object.keys(data).slice(-1)[0];
+              if (!info) {
+                var repoURL = "";
+                if (info.repository) {
+                  if (typeof info.repository == "string") repoURL = info.repository;
+                  else repoURL = info.repository.url;
+                }
+
+                resolutions[i].Target = {
+                 ToRepoCloneURL: repoURL,
+	          ToUnit: info.name,
+	          ToUnitType: "CommonJSPackage",
+	          ToVersionString: info.version
+                };
+              }
+            }
+
+            remaining--;
+            if (remaining == 0) {
+              // done resolving all
+              console.log(JSON.stringify(resolutions, null, 2));
+            }
+          });
+        });
+      });
+    });
+  })
+  .help("resolves dependencies of a CommonJS package");
+
+parser.command("graph")
+  .callback(function(opts) {
+    var stdin = process.stdin,
+    stdout = process.stdout,
+    inputChunks = [];
+
+    stdin.resume();
+    stdin.setEncoding("utf8");
+
+    stdin.on("data", function (chunk) {
+      inputChunks.push(chunk);
+    });
+
+    stdin.on("end", function () {
+      var inputJSON = inputChunks.join("");
+      var commonJSPackage = JSON.parse(inputJSON);
+
+      function graph() {
+        jsgAdapter.run(".", commonJSPackage, function(err, graphData) {
+          if (err) {
+            console.error("Graphing " + commonJSPackage.Name + " failed:", err);
+            process.exit(1);
+          }
+          console.log(JSON.stringify(graphData, null, 2));
+        });
+      }
+
+      // TODO(sqs) install npm deps if we're in a docker container, since we can't rely on
+      // them to have been installed locally. we need to account for the fact
+      // that we've mounted the source volume readonly, so we can't just run
+      // `npm install`.
+      if (process.env.IN_DOCKER_CONTAINER && commonJSPackage.Config.npmInstall) {
+        // running `npm install` in the /src dir fails because it's mounted
+        // readonly. but `npm install -g` in the /src dir does nothing. so we
+        // need to `npm install /src` from a dir other than /src to get `npm
+        // install` to do anything.
+
+
+        // TODO(sqs): only works for top-level package.jsons. we actually need
+        // "/src/${dirname(package.json path)", which also requires us to
+        // track the path to the package.json.
+        execFile("npm", ["install", "--ignore-scripts", "/src"], {cwd: "/"}, function(err, stdout, stderr) {
+          if (stderr) console.error(stderr);
+          if (stdout) console.error(stdout);
+          if (err) {
+            console.error("npm install in Docker container failed: ", err);
+            process.exit(1)
+          }
+          graph();
+        });
+
+
+        // It's apparently impossible to silence npm install output when using
+        // npm.install. :(
+        // var oldcwd = process.cwd();
+        // process.chdir(path.join(oldcwd, ".."));
+        // console.error("npm installing packages in Docker container...");
+        // npm.load({loglevel: "silent"}, function(err, npm) {
+        //   if (err) {
+        //     console.error("npm.load in Docker container failed: ", err);
+        //     process.exit(1)
+        //   }
+        //   // TODO(sqs): only works for top-level package.jsons. we actually need
+        //   // "/src/${dirname(package.json path)", which also requires us to
+        //   // track the path to the package.json.
+        //   npm.commands.install(["/src"], function(err,data) {
+        //     if (err) {
+        //       console.error("npm install in Docker container failed: ", err);
+        //       process.exit(1)
+        //     }
+        //     process.chdir(oldcwd);
+        //     graph();
+        //   });
+        // });
+      } else {
+        graph();
+      }
+    });
+  })
+  .help("graph a CommonJS package using jsg/tern");
+
+parser.parse();
diff --git a/.dockerignore b/.dockerignore
@@ -0,0 +1,5 @@
+.git
+testdata
+__old
+.srclib-cache
+
diff --git a/.gitignore b/.gitignore
@@ -0,0 +1 @@
+testdata/actual/
diff --git a/.gitmodules b/.gitmodules
@@ -0,0 +1,9 @@
+[submodule "testdata/case/javascript-nodejs-sample-0"]
+	path = testdata/case/javascript-nodejs-sample-0
+	url = https://github.com/sgtest/javascript-nodejs-sample-0.git
+[submodule "testdata/case/javascript-nodejs-xrefs-0"]
+	path = testdata/case/javascript-nodejs-xrefs-0
+	url = https://github.com/sgtest/javascript-nodejs-xrefs-0.git
+[submodule "testdata/case/minimal_nodejs_stdlib"]
+	path = testdata/case/minimal_nodejs_stdlib
+	url = https://github.com/sgtest/minimal_nodejs_stdlib.git
diff --git a/.travis.yml b/.travis.yml
@@ -0,0 +1,12 @@
+language: node_js
+
+node_js:
+  - "0.11"
+  - "0.10"
+
+install:
+  - sudo wget -NO /tmp/src.zip 'https://api.equinox.io/1/Applications/ap_BQxVz1iWMxmjQnbVGd85V58qz6/Updates/Asset/src-0.0.22.zip?os=linux&arch=amd64&channel=stable'
+  - sudo unzip /tmp/src*.zip -d /usr/local/bin
+  - sudo mv /usr/local/bin/src-* /usr/local/bin/src
+  - sudo chmod +x /usr/local/bin/src
+  - src toolchain add sourcegraph.com/sourcegraph/srclib-javascript
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -0,0 +1,71 @@
+# How to Contribute
+
+Sourcegraph projects are [BSD licensed](LICENSE) and accept contributions via
+GitHub pull requests. This document outlines some of the conventions on
+development workflow, commit message formatting, contact points and other
+resources to make it easier to get your contribution accepted.
+
+# Certificate of Origin
+
+By contributing to this project you agree to the Developer Certificate of Origin
+(DCO). This document was created by the Linux Kernel community and is a simple
+statement that you, as a contributor, have the legal right to make the
+contribution. See the [DCO](DCO) file for details.
+
+# Email and Chat
+
+The project currently uses the general Sourcegraph email list and IRC channel:
+- Email: [srclib-dev](https://groups.google.com/forum/#!forum/srclib-dev)
+- IRC: #[srclib](irc://irc.freenode.org:6667/#srclib) IRC channel on freenode.org
+
+Please avoid emailing maintainers found in the MAINTAINERS file directly. They
+are very busy and read the mailing lists.
+
+## Getting Started
+
+- Fork the repository on GitHub
+- Read the [README](README.md) for build and test instructions
+- Play with the project, submit bugs, submit patches!
+
+## Contribution Flow
+
+This is a rough outline of what a contributor's workflow looks like:
+
+- Create a topic branch from where you want to base your work (usually master).
+- Make commits of logical units.
+- Make sure your commit messages are in the proper format (see below).
+- Push your changes to a topic branch in your fork of the repository.
+- Make sure the tests pass, and add any new tests as appropriate.
+- Submit a pull request to the original repository.
+
+Thanks for your contributions!
+
+### Format of the Commit Message
+
+We follow a rough convention for commit messages that is designed to answer two
+questions: what changed and why. The subject line should feature the what and
+the body of the commit should describe the why.
+
+```
+scripts: add the test-cluster command
+
+this uses tmux to setup a test cluster that you can easily kill and
+start for debugging.
+
+Fixes #38
+```
+
+The format can be described more formally as follows:
+
+```
+<subsystem>: <what changed>
+<BLANK LINE>
+<why this change was made>
+<BLANK LINE>
+<footer>
+```
+
+The first line is the subject and should be no longer than 70 characters, the
+second line is always blank, and other lines should be wrapped at 80 characters.
+This allows the message to be easier to read on Sourcegraph and GitHub as well
+as in various git tools.
diff --git a/DCO b/DCO
@@ -0,0 +1,36 @@
+Developer Certificate of Origin
+Version 1.1
+
+Copyright (C) 2004, 2006 The Linux Foundation and its contributors.
+660 York Street, Suite 102,
+San Francisco, CA 94110 USA
+
+Everyone is permitted to copy and distribute verbatim copies of this
+license document, but changing it is not allowed.
+
+
+Developer's Certificate of Origin 1.1
+
+By making a contribution to this project, I certify that:
+
+(a) The contribution was created in whole or in part by me and I
+    have the right to submit it under the open source license
+    indicated in the file; or
+
+(b) The contribution is based upon previous work that, to the best
+    of my knowledge, is covered under an appropriate open source
+    license and I have the right under that license to submit that
+    work with modifications, whether created in whole or in part
+    by me, under the same open source license (unless I am
+    permitted to submit under a different license), as indicated
+    in the file; or
+
+(c) The contribution was provided directly to me by some other
+    person who certified (a), (b) or (c) and I have not modified
+    it.
+
+(d) I understand and agree that this project and the contribution
+    are public and that a record of the contribution (including all
+    personal information I submit with it, including my sign-off) is
+    maintained indefinitely and may be redistributed consistent with
+    this project or the open source license(s) involved.