diff --git a/.github/workflows/profiling.yml b/.github/workflows/profiling.yml new file mode 100644 index 0000000000..b4963c0889 --- /dev/null +++ b/.github/workflows/profiling.yml @@ -0,0 +1,53 @@ +name: Profiling + +on: + pull_request: + branches: + - og-develop + +permissions: + # deployments permission to deploy GitHub pages website + deployments: write + # contents permission to update profiling contents in gh-pages branch + contents: write + +concurrency: + group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.head_ref || github.sha }} + cancel-in-progress: true + +jobs: + profiling: + name: Speed Profiling + runs-on: [self-hosted, linux, gpu, dataset-enabled] + + defaults: + run: + shell: micromamba run -n omnigibson /bin/bash -leo pipefail {0} + + steps: + - name: Fix home + run: echo "HOME=/root" >> $GITHUB_ENV + + - name: Checkout source + uses: actions/checkout@v3 + + - name: Install dev requirements + run: pip install -r requirements-dev.txt + + - name: Install + run: pip install -e . + + - name: Run performance benchmark + run: bash scripts/profiling.sh + + - name: Store benchmark result + uses: benchmark-action/github-action-benchmark@v1 + with: + tool: 'customSmallerIsBetter' + output-file-path: output.json + benchmark-data-dir-path: profiling + fail-on-alert: true + alert-threshold: '200%' + github-token: ${{ secrets.GITHUB_TOKEN }} + comment-on-alert: true + auto-push: true diff --git a/omnigibson/systems/system_base.py b/omnigibson/systems/system_base.py index 0e06b8966d..d78dd863f8 100644 --- a/omnigibson/systems/system_base.py +++ b/omnigibson/systems/system_base.py @@ -1233,6 +1233,10 @@ def is_physical_particle_system(system_name): def get_system(system_name, force_active=True): # Make sure scene exists assert og.sim.scene is not None, "Cannot get systems until scene is imported!" + # Make sure prefixes preserve their double underscore + for prefix in SYSTEM_PREFIXES: + if f"{prefix}__" not in system_name: + system_name = system_name.replace(f"{prefix}_", f"{prefix}__") # If system_name is not in REGISTERED_SYSTEMS, create from metadata system = REGISTERED_SYSTEMS[system_name] if system_name in REGISTERED_SYSTEMS \ else _create_system_from_metadata(system_name=system_name) diff --git a/omnigibson/transition_rules.py b/omnigibson/transition_rules.py index 0b80ab5906..a6b5ba6160 100644 --- a/omnigibson/transition_rules.py +++ b/omnigibson/transition_rules.py @@ -799,9 +799,8 @@ def _generate_conditions(cls): @classmethod def transition(cls, object_candidates): objs_to_remove = [] - for diceable_obj in object_candidates["diceable"]: - system = get_system(f"diced_{diceable_obj.category}") + system = get_system(f"diced__{diceable_obj.category}") system.generate_particles_from_link(diceable_obj, diceable_obj.root_link, check_contact=False, use_visual_meshes=False) # Delete original object from stage. diff --git a/omnigibson/utils/profiling_utils.py b/omnigibson/utils/profiling_utils.py new file mode 100644 index 0000000000..71a3df9047 --- /dev/null +++ b/omnigibson/utils/profiling_utils.py @@ -0,0 +1,56 @@ +import gym +import omnigibson as og + +from time import time +from omnigibson.envs.env_base import Environment + +PROFILING_FIELDS = ["total time", "physics time", "render time", "non physics time", "get observation time", "task time", "action time"] + +class ProfilingEnv(Environment): + def step(self, action): + start = time() + # If the action is not a dictionary, convert into a dictionary + if not isinstance(action, dict) and not isinstance(action, gym.spaces.Dict): + action_dict = dict() + idx = 0 + for robot in self.robots: + action_dim = robot.action_dim + action_dict[robot.name] = action[idx: idx + action_dim] + idx += action_dim + else: + # Our inputted action is the action dictionary + action_dict = action + + # Iterate over all robots and apply actions + for robot in self.robots: + robot.apply_action(action_dict[robot.name]) + action_end = time() + # Run simulation step + # Possibly force playing + for i in range(og.sim.n_physics_timesteps_per_render): + super(type(og.sim), og.sim).step(render=False) + physics_end = time() + og.sim.render() + render_end = time() + og.sim._non_physics_step() + non_physics_end = time() + # Grab observations + obs = self.get_obs() + obs_end = time() + # Grab reward, done, and info, and populate with internal info, then get scene state + reward, done, info = self.task.step(self, action) + self._populate_info(info) + if done and self._automatic_reset: + # Add lost observation to our information dict, and reset + info["last_observation"] = obs + obs = self.reset() + # save scene state + self.scene.dump_state(serialized=True) + # Increment step + self._current_step += 1 + end = time() + ret = [end-start, physics_end-action_end, render_end-physics_end, non_physics_end-render_end, \ + obs_end-non_physics_end, end-obs_end, action_end-start] + if self._current_step % 100 == 0: + print("total time: {:.3f} ms, physics time: {:.3f} ms, render time: {:.3f} ms, non physics time: {:.3f} ms, get obs time: {:.3f} ms, task time: {:.3f} ms, action time: {:.3f} ms, ".format(*ret)) + return obs, reward, done, info, ret diff --git a/scripts/benchmark.css b/scripts/profiling.css similarity index 100% rename from scripts/benchmark.css rename to scripts/profiling.css diff --git a/scripts/benchmark.html b/scripts/profiling.html similarity index 98% rename from scripts/benchmark.html rename to scripts/profiling.html index 3d9f5c1a59..18eb6815c9 100644 --- a/scripts/benchmark.html +++ b/scripts/profiling.html @@ -3,7 +3,7 @@