mirror of
https://github.com/BigfootACA/arch-image-builder.git
synced 2024-09-21 10:01:20 +08:00
builder: component: pacman.py: add pacman mirrors select
Signed-off-by: BigfootACA <bigfoot@classfun.cn>
This commit is contained in:
parent
11ee7f3517
commit
e54f17a02f
21
README.md
21
README.md
@ -23,20 +23,21 @@ pacman -S qemu-user-static-binfmt
|
|||||||
## Example
|
## Example
|
||||||
|
|
||||||
```commandline
|
```commandline
|
||||||
python build.py -c target/ayn-odin2-sdcard,locale/zh-cn,desktop/gnome
|
python build.py -c target/ayn-odin2-sdcard,locale/zh-cn,desktop/gnome -m bfsu,tuna
|
||||||
```
|
```
|
||||||
|
|
||||||
## Options
|
## Options
|
||||||
|
|
||||||
| Option | Description |
|
| Option | Description |
|
||||||
|-------------------------------------|----------------------------------|
|
|-------------------------------------|------------------------------------|
|
||||||
| -C, --clean | Clean workspace before build |
|
| -C, --clean | Clean workspace before build |
|
||||||
| -p PRESET, --preset PRESET | Select preset to create package |
|
| -m MIRROR, --mirror MIRROR | Select mirror to download package |
|
||||||
| -c CONFIG, --config CONFIG | Select configs to build |
|
| -p PRESET, --preset PRESET | Select preset to create package |
|
||||||
| -o WORKSPACE, --workspace WORKSPACE | Set workspace for builder |
|
| -c CONFIG, --config CONFIG | Select configs to build |
|
||||||
| -d, --debug | Enable debug logging |
|
| -o WORKSPACE, --workspace WORKSPACE | Set workspace for builder |
|
||||||
| -G, --no-gpgcheck | Disable GPG check |
|
| -d, --debug | Enable debug logging |
|
||||||
| -r, --repack | Repack rootfs only |
|
| -G, --no-gpgcheck | Disable GPG check |
|
||||||
|
| -r, --repack | Repack rootfs only |
|
||||||
|
|
||||||
## License
|
## License
|
||||||
|
|
||||||
|
@ -4,8 +4,10 @@ import logging
|
|||||||
import shutil
|
import shutil
|
||||||
import libarchive
|
import libarchive
|
||||||
from logging import getLogger
|
from logging import getLogger
|
||||||
|
from builder.lib.serializable import SerializableDict
|
||||||
from builder.lib.context import ArchBuilderContext
|
from builder.lib.context import ArchBuilderContext
|
||||||
from builder.lib.config import ArchBuilderConfigError
|
from builder.lib.config import ArchBuilderConfigError
|
||||||
|
from builder.lib.subscript import resolve_simple_values
|
||||||
log = getLogger(__name__)
|
log = getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
@ -30,6 +32,51 @@ def progress_cb(target, percent, n, i):
|
|||||||
log.info(f"processing {target} ({i}/{n})")
|
log.info(f"processing {target} ({i}/{n})")
|
||||||
|
|
||||||
|
|
||||||
|
class PacmanRepoServer(SerializableDict):
|
||||||
|
url: str = None
|
||||||
|
name: str = None
|
||||||
|
mirror: bool = False
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
name: str = None,
|
||||||
|
url: str = None,
|
||||||
|
mirror: bool = None
|
||||||
|
):
|
||||||
|
if url is not None: self.url = url
|
||||||
|
if name is not None: self.name = name
|
||||||
|
if mirror is not None: self.mirror = mirror
|
||||||
|
|
||||||
|
|
||||||
|
class PacmanRepo(SerializableDict):
|
||||||
|
name: str = None
|
||||||
|
priority: int = 10000
|
||||||
|
servers: list[PacmanRepoServer] = None
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
name: str = None,
|
||||||
|
priority: int = None,
|
||||||
|
servers: list[PacmanRepoServer] = None
|
||||||
|
):
|
||||||
|
if name is not None: self.name = name
|
||||||
|
if priority is not None: self.priority = priority
|
||||||
|
if servers is not None: self.servers = servers
|
||||||
|
else: self.servers = []
|
||||||
|
|
||||||
|
def add_server(
|
||||||
|
self,
|
||||||
|
name: str = None,
|
||||||
|
url: str = None,
|
||||||
|
mirror: bool = None
|
||||||
|
):
|
||||||
|
self.servers.append(PacmanRepoServer(
|
||||||
|
name=name,
|
||||||
|
url=url,
|
||||||
|
mirror=mirror,
|
||||||
|
))
|
||||||
|
|
||||||
|
|
||||||
class Pacman:
|
class Pacman:
|
||||||
handle: pyalpm.Handle
|
handle: pyalpm.Handle
|
||||||
ctx: ArchBuilderContext
|
ctx: ArchBuilderContext
|
||||||
@ -37,17 +84,22 @@ class Pacman:
|
|||||||
databases: dict[str: pyalpm.DB]
|
databases: dict[str: pyalpm.DB]
|
||||||
config: dict
|
config: dict
|
||||||
caches: list[str]
|
caches: list[str]
|
||||||
|
repos: list[PacmanRepo]
|
||||||
|
|
||||||
def append_repos(self, lines: list[str]):
|
def append_repos(self, lines: list[str]):
|
||||||
"""
|
"""
|
||||||
Add all databases into config
|
Add all databases into config
|
||||||
"""
|
"""
|
||||||
for repo in self.databases:
|
for repo in self.repos:
|
||||||
db = self.databases[repo]
|
lines.append(f"[{repo.name}]\n")
|
||||||
lines.append(f"[{repo}]\n")
|
for server in repo.servers:
|
||||||
for server in db.servers:
|
if server.mirror:
|
||||||
log.debug(f"server {server}")
|
lines.append(f"# Mirror {server.name}\n")
|
||||||
lines.append(f"Server = {server}\n")
|
log.debug(f"use mirror {server.name} url {server.url}")
|
||||||
|
else:
|
||||||
|
lines.append("# Original Repo\n")
|
||||||
|
log.debug(f"use original repo url {server.url}")
|
||||||
|
lines.append(f"Server = {server.url}\n")
|
||||||
|
|
||||||
def append_config(self, lines: list[str]):
|
def append_config(self, lines: list[str]):
|
||||||
"""
|
"""
|
||||||
@ -127,54 +179,27 @@ class Pacman:
|
|||||||
ret = self.ctx.run_external(cmds)
|
ret = self.ctx.run_external(cmds)
|
||||||
if ret != 0: raise OSError(f"pacman failed with {ret}")
|
if ret != 0: raise OSError(f"pacman failed with {ret}")
|
||||||
|
|
||||||
def add_database(self, repo: dict):
|
|
||||||
"""
|
|
||||||
Add a database and update it
|
|
||||||
"""
|
|
||||||
def resolve(url: str) -> str:
|
|
||||||
"""
|
|
||||||
Replace pacman.conf variables
|
|
||||||
"""
|
|
||||||
return (url
|
|
||||||
.replace("$arch", self.ctx.tgt_arch)
|
|
||||||
.replace("$repo", name))
|
|
||||||
if "name" not in repo:
|
|
||||||
raise ArchBuilderConfigError("repo name not set")
|
|
||||||
name = repo["name"]
|
|
||||||
|
|
||||||
# never add local into database
|
|
||||||
if name == "local" or "/" in name:
|
|
||||||
raise ArchBuilderConfigError("bad repo name")
|
|
||||||
|
|
||||||
# register database
|
|
||||||
if name not in self.databases:
|
|
||||||
self.databases[name] = self.handle.register_syncdb(
|
|
||||||
name, pyalpm.SIG_DATABASE_MARGINAL_OK
|
|
||||||
)
|
|
||||||
db = self.databases[name]
|
|
||||||
|
|
||||||
# add databases servers
|
|
||||||
servers: list[str] = []
|
|
||||||
if "server" in repo:
|
|
||||||
servers.append(resolve(repo["server"]))
|
|
||||||
if "servers" in repo:
|
|
||||||
for server in repo["servers"]:
|
|
||||||
servers.append(resolve(server))
|
|
||||||
db.servers = servers
|
|
||||||
|
|
||||||
# update database now via pyalpm
|
|
||||||
log.info(f"updating database {name}")
|
|
||||||
db.update(False)
|
|
||||||
|
|
||||||
def load_databases(self):
|
def load_databases(self):
|
||||||
"""
|
"""
|
||||||
Add all databases and load them
|
Add all databases and load them
|
||||||
"""
|
"""
|
||||||
cfg = self.config
|
for mirror in self.repos:
|
||||||
if "repo" not in cfg:
|
# register database
|
||||||
raise ArchBuilderConfigError("no repos found in config")
|
if mirror.name not in self.databases:
|
||||||
for repo in cfg["repo"]:
|
self.databases[mirror.name] = self.handle.register_syncdb(
|
||||||
self.add_database(repo)
|
mirror.name, pyalpm.SIG_DATABASE_MARGINAL_OK
|
||||||
|
)
|
||||||
|
db = self.databases[mirror.name]
|
||||||
|
|
||||||
|
# add databases servers
|
||||||
|
servers: list[str] = []
|
||||||
|
for server in mirror.servers:
|
||||||
|
servers.append(server.url)
|
||||||
|
db.servers = servers
|
||||||
|
|
||||||
|
# update database now via pyalpm
|
||||||
|
log.info(f"updating database {mirror.name}")
|
||||||
|
db.update(False)
|
||||||
self.init_config()
|
self.init_config()
|
||||||
self.refresh()
|
self.refresh()
|
||||||
|
|
||||||
@ -231,6 +256,78 @@ class Pacman:
|
|||||||
os.makedirs(work_cache, mode=0o0755, exist_ok=True)
|
os.makedirs(work_cache, mode=0o0755, exist_ok=True)
|
||||||
os.makedirs(root_cache, mode=0o0755, exist_ok=True)
|
os.makedirs(root_cache, mode=0o0755, exist_ok=True)
|
||||||
|
|
||||||
|
def add_repo(self, repo: PacmanRepo):
|
||||||
|
if not repo or not repo.name or len(repo.servers) <= 0:
|
||||||
|
raise ArchBuilderConfigError("bad repo")
|
||||||
|
self.repos.append(repo)
|
||||||
|
self.repos.sort(key=lambda r: r.priority)
|
||||||
|
|
||||||
|
def init_repos(self):
|
||||||
|
"""
|
||||||
|
Initialize mirrors
|
||||||
|
"""
|
||||||
|
if "repo" not in self.config:
|
||||||
|
raise ArchBuilderConfigError("no repos found in config")
|
||||||
|
mirrors = self.ctx.get("mirrors", [])
|
||||||
|
for repo in self.config["repo"]:
|
||||||
|
if "name" not in repo:
|
||||||
|
raise ArchBuilderConfigError("repo name not set")
|
||||||
|
|
||||||
|
# never add local into database
|
||||||
|
if repo["name"] == "local" or "/" in repo["name"]:
|
||||||
|
raise ArchBuilderConfigError("bad repo name")
|
||||||
|
|
||||||
|
# create pacman repo instance
|
||||||
|
pacman_repo = PacmanRepo(name=repo["name"])
|
||||||
|
if "priority" in repo:
|
||||||
|
pacman_repo.priority = repo["priority"]
|
||||||
|
|
||||||
|
originals: list[str] = []
|
||||||
|
servers: list[str] = []
|
||||||
|
|
||||||
|
# add all original repo url
|
||||||
|
if "server" in repo: servers.append(repo["server"])
|
||||||
|
if "servers" in repo: servers.extend(repo["server"])
|
||||||
|
if len(servers) <= 0:
|
||||||
|
raise ArchBuilderConfigError("no any original repo url found")
|
||||||
|
|
||||||
|
# resolve original repo url
|
||||||
|
for server in servers:
|
||||||
|
originals.append(resolve_simple_values(server, {
|
||||||
|
"arch": self.ctx.tgt_arch,
|
||||||
|
"repo": repo["name"],
|
||||||
|
}))
|
||||||
|
|
||||||
|
# add repo mirror url
|
||||||
|
for mirror in mirrors:
|
||||||
|
if "name" not in mirror:
|
||||||
|
raise ArchBuilderConfigError("mirror name not set")
|
||||||
|
if "repos" not in mirror:
|
||||||
|
raise ArchBuilderConfigError("repos list not set")
|
||||||
|
for repo in mirror["repos"]:
|
||||||
|
if "original" not in repo:
|
||||||
|
raise ArchBuilderConfigError("original url not set")
|
||||||
|
if "mirror" not in repo:
|
||||||
|
raise ArchBuilderConfigError("mirror url not set")
|
||||||
|
for original in originals:
|
||||||
|
if original.startswith(repo["original"]):
|
||||||
|
path = original[len(repo["original"]):]
|
||||||
|
real_url = repo["mirror"] + path
|
||||||
|
pacman_repo.add_server(
|
||||||
|
name=mirror["name"],
|
||||||
|
url=real_url,
|
||||||
|
mirror=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
# add original url
|
||||||
|
for original in originals:
|
||||||
|
pacman_repo.add_server(
|
||||||
|
url=original,
|
||||||
|
mirror=False
|
||||||
|
)
|
||||||
|
|
||||||
|
self.add_repo(pacman_repo)
|
||||||
|
|
||||||
def __init__(self, ctx: ArchBuilderContext):
|
def __init__(self, ctx: ArchBuilderContext):
|
||||||
"""
|
"""
|
||||||
Initialize pacman context
|
Initialize pacman context
|
||||||
@ -250,7 +347,9 @@ class Pacman:
|
|||||||
self.handle.progresscb = progress_cb
|
self.handle.progresscb = progress_cb
|
||||||
self.databases = {}
|
self.databases = {}
|
||||||
self.caches = []
|
self.caches = []
|
||||||
|
self.repos = []
|
||||||
self.init_cache()
|
self.init_cache()
|
||||||
|
self.init_repos()
|
||||||
for cache in self.caches:
|
for cache in self.caches:
|
||||||
self.handle.add_cachedir(cache)
|
self.handle.add_cachedir(cache)
|
||||||
self.init_config()
|
self.init_config()
|
||||||
|
@ -18,6 +18,7 @@ def parse_arguments(ctx: ArchBuilderContext):
|
|||||||
parser.add_argument("-C", "--clean", help="Clean workspace before build", default=False, action='store_true')
|
parser.add_argument("-C", "--clean", help="Clean workspace before build", default=False, action='store_true')
|
||||||
parser.add_argument("-p", "--preset", help="Select preset to create package")
|
parser.add_argument("-p", "--preset", help="Select preset to create package")
|
||||||
parser.add_argument("-c", "--config", help="Select config to build", action='append')
|
parser.add_argument("-c", "--config", help="Select config to build", action='append')
|
||||||
|
parser.add_argument("-m", "--mirror", help="Select mirror to download package", action='append')
|
||||||
parser.add_argument("-o", "--workspace", help="Set workspace for builder", default=ctx.work)
|
parser.add_argument("-o", "--workspace", help="Set workspace for builder", default=ctx.work)
|
||||||
parser.add_argument("-d", "--debug", help="Enable debug logging", default=False, action='store_true')
|
parser.add_argument("-d", "--debug", help="Enable debug logging", default=False, action='store_true')
|
||||||
parser.add_argument("-G", "--no-gpgcheck", help="Disable GPG check", default=False, action='store_true')
|
parser.add_argument("-G", "--no-gpgcheck", help="Disable GPG check", default=False, action='store_true')
|
||||||
@ -45,6 +46,10 @@ def parse_arguments(ctx: ArchBuilderContext):
|
|||||||
pcfgs = ctx.get("package.configs", [])
|
pcfgs = ctx.get("package.configs", [])
|
||||||
configs.extend(pcfgs)
|
configs.extend(pcfgs)
|
||||||
|
|
||||||
|
if args.mirror:
|
||||||
|
for mirror in args.mirror:
|
||||||
|
configs.extend([f"mirrors/{name}" for name in mirror.split(",")])
|
||||||
|
|
||||||
# load and populate configs
|
# load and populate configs
|
||||||
config.load_configs(ctx, configs)
|
config.load_configs(ctx, configs)
|
||||||
config.populate_config(ctx)
|
config.populate_config(ctx)
|
||||||
|
Loading…
Reference in New Issue
Block a user