AlignmentResearch · tigist-far · Jul 8, 2026 · Jul 22, 2026 · Sep 1, 2026 · Sep 1, 2026
diff --git a/FARAI.md b/FARAI.md
@@ -0,0 +1,23 @@
+# FAR.AI build of SkyPilot
+
+Branch `farai/v0.13.0` is upstream `v0.13.0` plus the commits listed by
+`git log v0.13.0..` — each one a self-contained fix that the FAR.AI clusters need and that
+is written to be upstreamable as-is. The `nrl` launcher in
+[AlignmentResearch/nemo-rl](https://github.com/AlignmentResearch/nemo-rl) pins a tag of this
+branch as its `skypilot` dependency, so `sky --version` on a launcher install reports the
+local version label (`0.13.0+farai.1`) and a stock install does not.
+
+## Cutting a new build
+
+1. Commit the change on this branch (one fix per commit, subject in upstream's `[Area]` style).
+2. Bump `__version__` in `sky/__init__.py` to the next `0.13.0+farai.N`.
+3. Tag it: `git tag -a v0.13.0-farai.N -m 'FAR.AI build N of SkyPilot 0.13.0'` and push the
+   branch and the tag.
+4. Point `launcher/pyproject.toml` in nemo-rl at the new tag and relock (`uv lock` in
+   `launcher/`, then at the repo root).
+
+## Moving to a new upstream release
+
+Create `farai/v<release>` from the upstream tag, cherry-pick the commits from the previous
+branch (dropping any that upstream now carries), vendor `sky/dashboard/out` from the new PyPI
+wheel, and cut build 1 as above.
diff --git a/sky/__init__.py b/sky/__init__.py
@@ -45,7 +45,7 @@ def _get_git_commit():
 
 
 __commit__ = _get_git_commit()
-__version__ = '1.0.0-dev0'
+__version__ = '0.13.0+farai.1'
 __root_dir__ = directory_utils.get_sky_dir()
 
 

diff --git a/sky/backends/task_codegen.py b/sky/backends/task_codegen.py
@@ -563,7 +563,10 @@ def check_ip():
                 ])
 
                 cluster_ips_to_node_id = {{ip: i for i, ip in enumerate({stable_cluster_internal_ips!r})}}
-                job_ip_rank_list = sorted(gang_scheduling_id_to_ip, key=cluster_ips_to_node_id.get)
+                # --- nemo-rl: tolerate multi-NIC Ray IP mismatches ---
+                job_ip_rank_list = sorted(
+                    gang_scheduling_id_to_ip,
+                    key=lambda ip: (cluster_ips_to_node_id.get(ip, len(cluster_ips_to_node_id)), ip))
                 job_ip_rank_map = {{ip: i for i, ip in enumerate(job_ip_rank_list)}}
                 job_ip_list_str = '\\n'.join(job_ip_rank_list)
                 """),
@@ -656,8 +659,9 @@ def _add_ray_task(self,
                 name_str = '{task_name},' if {task_name!r} != None else 'task,'
                 log_path = os.path.expanduser(os.path.join({log_dir!r}, 'run.log'))
             else: # Single-node or multi-node task on multi-node cluster
-                idx_in_cluster = cluster_ips_to_node_id[ip]
-                if cluster_ips_to_node_id[ip] == 0:
+                # --- nemo-rl: name an unmapped multi-NIC rank safely ---
+                idx_in_cluster = cluster_ips_to_node_id.get(ip, len(cluster_ips_to_node_id) + {gang_scheduling_id!r})
+                if idx_in_cluster == 0:
                     node_name = 'head'
                 else:
                     node_name = f'worker{{idx_in_cluster}}'

diff --git a/sky/dashboard/out/404.html b/sky/dashboard/out/404.html
@@ -0,0 +1 @@
+<!DOCTYPE html><html><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width"/><meta name="next-head-count" content="2"/><link rel="preload" href="/dashboard/_next/static/css/b9d558a5c9adf7e4.css" as="style"/><link rel="stylesheet" href="/dashboard/_next/static/css/b9d558a5c9adf7e4.css" data-n-g=""/><noscript data-n-css=""></noscript><script defer="" nomodule="" src="/dashboard/_next/static/chunks/polyfills-78c92fac7aa8fdd8.js"></script><script src="/dashboard/_next/static/chunks/webpack-45982c64a14663b4.js" defer=""></script><script src="/dashboard/_next/static/chunks/framework-5c7da2e0e187e3b5.js" defer=""></script><script src="/dashboard/_next/static/chunks/main-e344280b08a651ae.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/_app-e81a18ce7bb2b32b.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/_error-1be831200e60c5c0.js" defer=""></script><script src="/dashboard/_next/static/CLqXjw8HCPK4_OiqauvBK/_buildManifest.js" defer=""></script><script src="/dashboard/_next/static/CLqXjw8HCPK4_OiqauvBK/_ssgManifest.js" defer=""></script></head><body><div id="__next"></div><script id="__NEXT_DATA__" type="application/json">{"props":{"pageProps":{"statusCode":404}},"page":"/_error","query":{},"buildId":"CLqXjw8HCPK4_OiqauvBK","assetPrefix":"/dashboard","nextExport":true,"isFallback":false,"gip":true,"scriptLoader":[]}</script></body></html>
diff --git a/sky/dashboard/out/[...path].html b/sky/dashboard/out/[...path].html
@@ -0,0 +1 @@
+<!DOCTYPE html><html><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width"/><meta name="next-head-count" content="2"/><link rel="preload" href="/dashboard/_next/static/css/b9d558a5c9adf7e4.css" as="style"/><link rel="stylesheet" href="/dashboard/_next/static/css/b9d558a5c9adf7e4.css" data-n-g=""/><noscript data-n-css=""></noscript><script defer="" nomodule="" src="/dashboard/_next/static/chunks/polyfills-78c92fac7aa8fdd8.js"></script><script src="/dashboard/_next/static/chunks/webpack-45982c64a14663b4.js" defer=""></script><script src="/dashboard/_next/static/chunks/framework-5c7da2e0e187e3b5.js" defer=""></script><script src="/dashboard/_next/static/chunks/main-e344280b08a651ae.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/_app-e81a18ce7bb2b32b.js" defer=""></script><script src="/dashboard/_next/static/chunks/739-e8580e965e4977a9.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/%5B...path%5D-1543afc4e1acf875.js" defer=""></script><script src="/dashboard/_next/static/CLqXjw8HCPK4_OiqauvBK/_buildManifest.js" defer=""></script><script src="/dashboard/_next/static/CLqXjw8HCPK4_OiqauvBK/_ssgManifest.js" defer=""></script></head><body><div id="__next"></div><script id="__NEXT_DATA__" type="application/json">{"props":{"pageProps":{}},"page":"/[...path]","query":{},"buildId":"CLqXjw8HCPK4_OiqauvBK","assetPrefix":"/dashboard","nextExport":true,"autoExport":true,"isFallback":false,"scriptLoader":[]}</script></body></html>
diff --git a/sky/dashboard/out/_next/static/CLqXjw8HCPK4_OiqauvBK/_buildManifest.js b/sky/dashboard/out/_next/static/CLqXjw8HCPK4_OiqauvBK/_buildManifest.js
diff --git a/sky/dashboard/out/_next/static/CLqXjw8HCPK4_OiqauvBK/_ssgManifest.js b/sky/dashboard/out/_next/static/CLqXjw8HCPK4_OiqauvBK/_ssgManifest.js
@@ -0,0 +1 @@
+self.__SSG_MANIFEST=new Set([]);self.__SSG_MANIFEST_CB&&self.__SSG_MANIFEST_CB()
diff --git a/sky/dashboard/out/_next/static/chunks/111.ee0ae633bd4cb952.js b/sky/dashboard/out/_next/static/chunks/111.ee0ae633bd4cb952.js
diff --git a/sky/dashboard/out/_next/static/chunks/192.869abb892841fabb.js b/sky/dashboard/out/_next/static/chunks/192.869abb892841fabb.js
diff --git a/sky/dashboard/out/_next/static/chunks/195-d38091b2de5396cf.js b/sky/dashboard/out/_next/static/chunks/195-d38091b2de5396cf.js
diff --git a/sky/dashboard/out/_next/static/chunks/256-426bc47289752b8f.js b/sky/dashboard/out/_next/static/chunks/256-426bc47289752b8f.js
diff --git a/sky/dashboard/out/_next/static/chunks/37-1ceb6ddb802bc6c9.js b/sky/dashboard/out/_next/static/chunks/37-1ceb6ddb802bc6c9.js
diff --git a/sky/dashboard/out/_next/static/chunks/400.f86a54d1da11a290.js b/sky/dashboard/out/_next/static/chunks/400.f86a54d1da11a290.js
diff --git a/sky/dashboard/out/_next/static/chunks/45-c7883b1e5aaf1496.js b/sky/dashboard/out/_next/static/chunks/45-c7883b1e5aaf1496.js
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		<!DOCTYPE html><html><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width"/><meta name="next-head-count" content="2"/><link rel="preload" href="/dashboard/_next/static/css/b9d558a5c9adf7e4.css" as="style"/><link rel="stylesheet" href="/dashboard/_next/static/css/b9d558a5c9adf7e4.css" data-n-g=""/><noscript data-n-css=""></noscript><script defer="" nomodule="" src="/dashboard/_next/static/chunks/polyfills-78c92fac7aa8fdd8.js"></script><script src="/dashboard/_next/static/chunks/webpack-45982c64a14663b4.js" defer=""></script><script src="/dashboard/_next/static/chunks/framework-5c7da2e0e187e3b5.js" defer=""></script><script src="/dashboard/_next/static/chunks/main-e344280b08a651ae.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/_app-e81a18ce7bb2b32b.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/_error-1be831200e60c5c0.js" defer=""></script><script src="/dashboard/_next/static/CLqXjw8HCPK4_OiqauvBK/_buildManifest.js" defer=""></script><script src="/dashboard/_next/static/CLqXjw8HCPK4_OiqauvBK/_ssgManifest.js" defer=""></script></head><body><div id="__next"></div><script id="__NEXT_DATA__" type="application/json">{"props":{"pageProps":{"statusCode":404}},"page":"/_error","query":{},"buildId":"CLqXjw8HCPK4_OiqauvBK","assetPrefix":"/dashboard","nextExport":true,"isFallback":false,"gip":true,"scriptLoader":[]}</script></body></html>
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		self.__SSG_MANIFEST=new Set([]);self.__SSG_MANIFEST_CB&&self.__SSG_MANIFEST_CB()