Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion base-containers/base/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ RUN dnf clean metadata && \
dnf -y install epel-release && \
dnf -y upgrade && \
dnf -y install python38 python38-pip python38-devel zip unzip tar sed openssh-server openssl bind-utils iproute file jq procps-ng man curl net-tools screen nano bc && \
pip3.8 install msgpack pyzmq jinja2 PyYAML timeout-decorator ipython mypy && \
pip3.8 install msgpack pyzmq jinja2 PyYAML timeout-decorator ipython mypy pydevd-pycharm && \
dnf clean all

# Allow to run commands
Expand Down
19 changes: 19 additions & 0 deletions base-containers/base/bin/INGInious
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,22 @@ import asyncio
import struct
import zmq
import zmq.asyncio
import pydevd_pycharm


debug_server_unavailable = False
if os.environ["DEBUGGER"] == "True":
print("Starting in debug mode, waiting for pycharm to connect...")
try:
pydevd_pycharm.settrace(
'host.docker.internal',
port=5678,
stdout_to_server=True,
stderr_to_server=True,
suspend=False
)
except Exception as e:
debug_server_unavailable = True
Comment on lines +37 to +38
Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It seems it already prints a stack trace so maybe there already is a try/except behind this one which becomes unecessary. But not sure...

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

There is indeed a stack trace :

Could not connect to host.docker.internal: 5678
Traceback (most recent call last):
  File "/usr/local/lib/python3.8/site-packages/_pydevd_bundle/pydevd_comm.py", line 443, in start_client
    s.connect((host, port))
ConnectionRefusedError: [Errno 111] Connection refused

But I don't get wan't you want me to do. Should I remove the print ?

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

There is no need for duplicate messages. Either catch the first exception if possible, or simply remove the second if it this not stop execution.



ELF_MAGIC = b"\x7F\x45\x4C\x46"
Expand Down Expand Up @@ -395,6 +411,9 @@ class INGIniousMainRunner:
inginious_container_api.feedback.set_global_feedback('[SSH Debug] Nobody connected to the SSH server.')
stdout, stderr = b"", b""

if os.environ["DEBUGGER"] and debug_server_unavailable:
stderr += b"[DEBUG] WARNING : Debug server was unavailable.\n"

# Produce feedback
feedback = inginious_container_api.feedback.get_feedback()
if not feedback:
Expand Down
1 change: 1 addition & 0 deletions configuration.example.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ local-config:
# debug_host: localhost # host that will be indicated to users when they want to use the task remote debug feature. Automatically guessed by
# default
debug_ports: 64000-64100 #port range use for remote debug feature
debugger: true # for IDE debugging, uses port 5678

# MongoDB options
mongo_opt:
Expand Down
4 changes: 4 additions & 0 deletions doc/admin_doc/commands_doc/inginious-agent-docker.rst
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,10 @@ INGInious to use a local backend, it is automatically run by ``inginious-webapp`

Range of port for job remote debugging. By default it is 64120-64130

.. option:: --debugger

Enables container debug through IDE. Only supports Pycharm's debugging for the moment by using a python debug server using port 5678.

.. option:: --tmpdir TMPDIR

Path to a directory where the agent can store information,
Expand Down
3 changes: 3 additions & 0 deletions doc/admin_doc/install_doc/config_reference.rst
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,9 @@ The different entries are :
``tmp_dir``
A directory whose absolute path must be available by the docker daemon and INGInious at the same time. By default, it is ``./agent_tmp``.

``debugger``
Enables container debugging if set to ``true``. It is available only for Pycharm's debugging with a python debug server using port 5678. It is disabled by default.

``log_level``
Can be set to ``INFO``, ``WARN``, or ``DEBUG``. Specifies the logging verbosity.

Expand Down
11 changes: 10 additions & 1 deletion doc/teacher_doc/testing.rst
Original file line number Diff line number Diff line change
Expand Up @@ -38,4 +38,13 @@ Debugging bash scripts (run.sh)
````````````````````````````````
As opposed to python scripts, INGInious functions are exposed through the
environment so there is no need to use a particular loader. ``source run.sh`` might be a better
choice than running the script in a subshell to observe the state of variables after execution.
choice than running the script in a subshell to observe the state of variables after execution.

Container debugging through an IDE
------------------------------
INGInious also supports container debugging through an IDE. This is currently only supported for the INGInious
script and other scripts/APIs in the container by using Pycharm's debugging. You need to create a *Python Debug Server*
in Pycharm Run/Debug configurations, listening on port 5678, and add one or several *Path Mapping* between the local
and remote files to enable checkpoints.
To enable this feature, you need to start the agent with the ``--debugger`` option or start the webapp with the
``debugger`` config in your configuration.yaml.
14 changes: 10 additions & 4 deletions inginious/agent/docker_agent/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,7 @@ class DockerRunningStudentContainer:

class DockerAgent(Agent):
def __init__(self, context, backend_addr, friendly_name, concurrency,
address_host=None, external_ports=None, tmp_dir="./agent_tmp", runtimes=None, ssh_allowed=False):
address_host=None, external_ports=None, debugger=False, tmp_dir="./agent_tmp", runtimes=None, ssh_allowed=False):
"""
:param context: ZeroMQ context for this process
:param backend_addr: address of the backend (for example, "tcp://127.0.0.1:2222")
Expand Down Expand Up @@ -94,6 +94,9 @@ def __init__(self, context, backend_addr, friendly_name, concurrency,
self._address_host = address_host
self._external_ports = set(external_ports) if external_ports is not None else set()

# IDE debugging for grading containers
self._debugger = debugger

# Async proxy to os
self._aos = AsyncProxy(os)
self._ashutil = AsyncProxy(shutil)
Expand Down Expand Up @@ -289,7 +292,7 @@ def __new_job_sync(self, message: BackendNewJob, future_results):
environment_name = message.environment

try:
enable_network = message.environment_parameters.get("network_grading", False)
enable_network = message.environment_parameters.get("network_grading", False) or self._debugger
limits = message.environment_parameters.get("limits", {})
time_limit = int(limits.get("time", 30))
hard_time_limit = int(limits.get("hard_time", None) or time_limit * 3)
Expand Down Expand Up @@ -331,7 +334,7 @@ def __new_job_sync(self, message: BackendNewJob, future_results):
ports_needed.append(22)

ports = {}
if len(ports_needed) > 0: # if ssh_debug, put time limits to 30 min.
if len(ports_needed) > 0 or self._debugger: # if ssh_debug or container debug, put time limits to 30 min.
time_limit = 30 * 60
hard_time_limit = 30 * 60
for p in ports_needed:
Expand Down Expand Up @@ -387,7 +390,7 @@ def __new_job_sync(self, message: BackendNewJob, future_results):

# Run the container
try:
container_id = self._docker.sync.create_container(environment, enable_network, mem_limit, task_path,
container_id = self._docker.sync.create_container(environment, enable_network, self._debugger, mem_limit, task_path,
sockets_path, course_common_path,
course_common_student_path,
self.__get_fd_limit(), runtime,
Expand Down Expand Up @@ -888,6 +891,9 @@ async def close_and_delete(student_container_id=student_container_id_loop):
try:
return_value = await info.future_results

if return_value is None:
raise Exception("Grading container did not return any result.")

# Accepted types for return dict
accepted_types = {"stdout": str, "stderr": str, "result": str, "text": str, "grade": float,
"problems": dict, "custom": dict, "tests": dict, "state": str, "archive": str}
Expand Down
4 changes: 3 additions & 1 deletion inginious/agent/docker_agent/_docker_interface.py
Original file line number Diff line number Diff line change
Expand Up @@ -109,7 +109,7 @@ def get_host_ip(self, env_with_dig='ingi/inginious-c-default'):
except:
return None

def create_container(self, image, network_grading, mem_limit, task_path, sockets_path,
def create_container(self, image, network_grading, debugger, mem_limit, task_path, sockets_path,
course_common_path, course_common_student_path, fd_limit, runtime: str, ports=None):
"""
Creates a container.
Expand Down Expand Up @@ -143,6 +143,8 @@ def create_container(self, image, network_grading, mem_limit, task_path, sockets
oom_kill_disable=True,
network_mode=("bridge" if (network_grading or len(ports) > 0) else 'none'),
ports=ports,
extra_hosts={"host.docker.internal": "host-gateway"},
environment={"DEBUGGER" : debugger},
volumes={
task_path: {'bind': '/task'},
sockets_path: {'bind': '/sockets'},
Expand Down
3 changes: 2 additions & 1 deletion inginious/frontend/arch_helper.py
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,7 @@ def create_arch(configuration, context):
concurrency = local_config.get("concurrency", multiprocessing.cpu_count())
debug_host = local_config.get("debug_host", None)
debug_ports = local_config.get("debug_ports", None)
debugger = local_config.get("debugger", False)
tmp_dir = local_config.get("tmp_dir", "./agent_tmp")

if debug_ports is not None:
Expand All @@ -92,7 +93,7 @@ def create_arch(configuration, context):

client = Client(context, "inproc://backend_client")
backend = Backend(context, "inproc://backend_agent", "inproc://backend_client")
agent_docker = DockerAgent(context, "inproc://backend_agent", "Docker - Local agent", concurrency, debug_host, debug_ports, tmp_dir, ssh_allowed=True)
agent_docker = DockerAgent(context, "inproc://backend_agent", "Docker - Local agent", concurrency, debug_host, debug_ports, debugger, tmp_dir, ssh_allowed=True)
agent_mcq = MCQAgent(context, "inproc://backend_agent", "MCQ - Local agent", 1)

asyncio.ensure_future(_restart_on_cancel(logger, agent_docker))
Expand Down
4 changes: 3 additions & 1 deletion inginious/scripts/agent_docker.py
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,8 @@ def main():
parser.add_argument("--debug-host", help="Host used for job remote debugging. Should be an IP or an hostname. If not filled in, "
"it will be automatically guessed", default=None, type=str)
parser.add_argument("--debug-ports", help="Range of port for job remote debugging. By default it is 64120-64130", type=check_range, default="64120-64130")
parser.add_argument("--debugger", help="Enables container debug through IDE. Only supports Pycharm's debugging for the moment by using a python debug server using port 5678.",
action="store_true")
parser.add_argument("--tmpdir", help="Path to a directory where the agent can store information, such as caches. Defaults to ./agent_data",
default="./agent_data")
parser.add_argument("--concurrency", help="Maximal number of jobs that can run concurrently on this agent. By default, it is the two times the "
Expand Down Expand Up @@ -125,7 +127,7 @@ def main():

# Create agent
agent = DockerAgent(context, args.backend, args.friendly_name, args.concurrency,
address_host=args.debug_host, external_ports=args.debug_ports, tmp_dir=args.tmpdir,
address_host=args.debug_host, external_ports=args.debug_ports, debugger=args.debugger, tmp_dir=args.tmpdir,
runtimes=args.runtime, ssh_allowed=args.ssh)

# Run!
Expand Down
Loading