xtask_lib/tasks/util/
thirdparty.rs

1//! Thirdparty library download and build helpers.
2//!
3//! Library source URLs are defined as constants so Renovate can track them.
4//!
5//! ## Download
6//!
7//! [`download_libraries`] fetches each library's source.  Most libraries are
8//! downloaded as tarballs and extracted with the top-level directory stripped.
9//! Libraries that use git submodules (currently freetype2) are cloned with
10//! `--recurse-submodules` so submodule contents are always present.  If the
11//! cloned source ships an `autogen.sh` script, it is run immediately after
12//! cloning to generate the `configure` script that `build-kobo.sh` expects.
13//!
14//! ## Build
15//!
16//! [`build_libraries`] iterates over the packages in dependency order, applies
17//! `kobo.patch` if present, then invokes each library's own `build-kobo.sh`
18//! script.
19
20use std::path::Path;
21
22use anyhow::{Context, Result, bail};
23
24use super::{cmd, fs, http};
25
26/// Base names of all thirdparty shared libraries.
27///
28/// SONAMEs are discovered at runtime via `arm-linux-gnueabihf-readelf -d`
29/// because upstream libraries do not follow a consistent ABI versioning scheme.
30pub const SONAMES: &[&str] = &[
31    "libz.so",
32    "libbz2.so",
33    "libpng16.so",
34    "libjpeg.so",
35    "libopenjp2.so",
36    "libjbig2dec.so",
37    "libfreetype.so",
38    "libharfbuzz.so",
39    "libgumbo.so",
40    "libdjvulibre.so",
41    "libmupdf.so",
42];
43
44/// Returns the SONAME of `lib` in `libs_dir`.
45///
46/// When the library file exists, `arm-linux-gnueabihf-readelf -d` is used to
47/// extract the SONAME from the binary. When only a versioned file exists
48/// (e.g. `libz.so.1.2.13` without `libz.so`), the versioned filename is
49/// returned directly.
50///
51/// # Errors
52///
53/// Returns an error if `arm-linux-gnueabihf-readelf` fails or the SONAME
54/// cannot be determined.
55pub fn soname(libs_dir: &Path, lib: &str) -> Result<String> {
56    let so_path = libs_dir.join(lib);
57    if so_path.exists() {
58        let so_path_str = so_path
59            .to_str()
60            .with_context(|| format!("shared library path is not valid UTF-8: {so_path:?}"))?;
61        let output = cmd::output(
62            "arm-linux-gnueabihf-readelf",
63            &["-d", so_path_str],
64            libs_dir,
65            &[],
66        )?;
67        let soname = output
68            .lines()
69            .find(|line| line.contains("SONAME"))
70            .and_then(|line| line.split_whitespace().last())
71            .map(|token| {
72                token
73                    .trim_start_matches('[')
74                    .trim_end_matches(']')
75                    .to_string()
76            })
77            .with_context(|| format!("failed to find SONAME in readelf output for {lib}"))?;
78        Ok(soname)
79    } else {
80        let prefix = format!("{}.", lib);
81        let matching: Vec<_> = std::fs::read_dir(libs_dir)?
82            .filter_map(|e| e.ok())
83            .filter(|e| e.file_name().to_string_lossy().starts_with(&prefix))
84            .collect();
85
86        match matching.len() {
87            1 => Ok(matching[0].file_name().to_string_lossy().into_owned()),
88            0 => bail!(
89                "no versioned file found for {} in {}",
90                lib,
91                libs_dir.display()
92            ),
93            _ => bail!(
94                "multiple versioned files found for {} in {}",
95                lib,
96                libs_dir.display()
97            ),
98        }
99    }
100}
101
102/// Version strings for thirdparty libraries tracked by Renovate.
103///
104/// Each version constant is the single source of truth — the download URL is
105/// derived from it at call time in [`library_source`].  A Renovate regex manager
106/// in `renovate.json` matches these constants and opens PRs when new releases
107/// are available.
108///
109/// # TODO
110///
111/// Add Renovate regex managers for the remaining URL constants below so that
112/// all thirdparty dependency updates are tracked automatically.  The
113/// following are now tracked via VERSION constants: openjpeg, harfbuzz, gumbo.
114/// Their SONAMEs are discovered at build time via `readelf -d` rather than
115/// hardcoded.  Remaining: bzip2, jbig2dec, mupdf.
116pub const ZLIB_VERSION: &str = "1.3.2";
117pub const LIBPNG_VERSION: &str = "1.6.53";
118pub const DJVULIBRE_VERSION: &str = "3.5.30";
119/// IJG libjpeg version tracked via the libjpeg-turbo `jpeg-<version>` tag mirror.
120pub const LIBJPEG_VERSION: &str = "10";
121
122pub const BZIP2_URL: &str = "https://sourceware.org/pub/bzip2/bzip2-1.0.8.tar.gz";
123/// OpenJPEG version, derived from the archive URL.
124pub const OPENJPEG_VERSION: &str = "2.5.4";
125pub const JBIG2DEC_URL: &str =
126    "https://github.com/ArtifexSoftware/jbig2dec/releases/download/0.20/jbig2dec-0.20.tar.gz";
127/// FreeType version, cloned from `freetype/freetype` at tag `VER-X-Y-Z`.
128///
129/// Tracked by Renovate via the `github-tags` datasource with
130/// `extractVersionTemplate: "^VER-(?<version>.+)$"`.  freetype2 is cloned
131/// rather than downloaded as a tarball because its build system requires the
132/// `nyorain/dlg` git submodule, which is absent from archive tarballs.
133pub const FREETYPE2_VERSION: &str = "2.14.1";
134/// HarfBuzz version, derived from the archive URL.
135pub const HARFBUZZ_VERSION: &str = "14.2.0";
136/// Gumbo version, derived from the archive URL.
137pub const GUMBO_VERSION: &str = "0.10.1";
138
139pub const MUPDF_URL: &str = "https://casper.mupdf.com/downloads/archive/mupdf-1.27.0-source.tar.gz";
140
141/// All libraries in dependency order for building.
142const LIBRARY_NAMES: &[&str] = &[
143    "zlib",
144    "bzip2",
145    "libpng",
146    "libjpeg",
147    "openjpeg",
148    "jbig2dec",
149    "freetype2",
150    "harfbuzz",
151    "gumbo",
152    "djvulibre",
153    "mupdf",
154];
155
156/// Describes how a thirdparty library's source is obtained.
157pub enum LibrarySource {
158    /// Download a tarball and extract it with the top-level directory stripped.
159    Tarball(String),
160    /// Clone a git repository at a specific tag, recursing into submodules.
161    Git { repo: String, tag: String },
162}
163
164/// Returns the source descriptor for a named library.
165///
166/// # Errors
167///
168/// Returns an error if `name` is not a known library.
169pub fn library_source(name: &str) -> Result<LibrarySource> {
170    match name {
171        "zlib" => Ok(LibrarySource::Tarball(format!(
172            "https://github.com/madler/zlib/releases/download/v{v}/zlib-{v}.tar.gz",
173            v = ZLIB_VERSION
174        ))),
175        "bzip2" => Ok(LibrarySource::Tarball(BZIP2_URL.to_owned())),
176        "libpng" => Ok(LibrarySource::Tarball(format!(
177            "https://github.com/pnggroup/libpng/archive/refs/tags/v{v}.tar.gz",
178            v = LIBPNG_VERSION
179        ))),
180        "libjpeg" => Ok(LibrarySource::Tarball(format!(
181            "https://github.com/libjpeg-turbo/libjpeg-turbo/archive/refs/tags/jpeg-{v}.tar.gz",
182            v = LIBJPEG_VERSION
183        ))),
184        "openjpeg" => Ok(LibrarySource::Tarball(format!(
185            "https://github.com/uclouvain/openjpeg/archive/v{v}.tar.gz",
186            v = OPENJPEG_VERSION
187        ))),
188        "jbig2dec" => Ok(LibrarySource::Tarball(JBIG2DEC_URL.to_owned())),
189        "freetype2" => Ok(LibrarySource::Git {
190            repo: "https://github.com/freetype/freetype".to_owned(),
191            tag: format!("VER-{}", FREETYPE2_VERSION.replace('.', "-")),
192        }),
193        "harfbuzz" => Ok(LibrarySource::Tarball(format!(
194            "https://github.com/harfbuzz/harfbuzz/archive/{v}.tar.gz",
195            v = HARFBUZZ_VERSION
196        ))),
197        "gumbo" => Ok(LibrarySource::Tarball(format!(
198            "https://github.com/google/gumbo-parser/archive/v{v}.tar.gz",
199            v = GUMBO_VERSION
200        ))),
201        "djvulibre" => Ok(LibrarySource::Tarball(format!(
202            "https://github.com/barak/djvulibre/archive/refs/tags/release.{v}.tar.gz",
203            v = DJVULIBRE_VERSION
204        ))),
205        "mupdf" => Ok(LibrarySource::Tarball(MUPDF_URL.to_owned())),
206        _ => bail!("unknown thirdparty library: {name}"),
207    }
208}
209
210/// Downloads source for the given libraries into `thirdparty/`.
211///
212/// When `names` is empty all libraries are downloaded.  Tarballs are extracted
213/// with the top-level directory stripped.  Libraries with a [`LibrarySource::Git`]
214/// source are cloned with `--recurse-submodules` so submodule contents are
215/// always present.
216///
217/// Skips libraries with persisted marker files:
218/// - source-ready marker ([`SOURCE_READY_MARKER`])
219/// - built marker ([`BUILT_MARKER`])
220///
221/// This avoids fragile file-heuristic detection across heterogeneous upstream
222/// source trees.
223///
224/// # Errors
225///
226/// Returns an error if any download, extraction, or clone fails.
227pub fn download_libraries(thirdparty_dir: &Path, names: &[&str]) -> Result<()> {
228    let targets: Vec<&str> = if names.is_empty() {
229        LIBRARY_NAMES.to_vec()
230    } else {
231        names.to_vec()
232    };
233
234    for name in targets {
235        let dest_dir = thirdparty_dir.join(name);
236
237        if is_source_ready(&dest_dir) || is_built(&dest_dir) {
238            println!("Skipping {name} (source ready)…");
239            continue;
240        }
241
242        println!("Downloading {name}…");
243
244        match library_source(name)? {
245            LibrarySource::Tarball(url) => {
246                let tarball = thirdparty_dir.join(format!("{name}.tgz"));
247
248                if dest_dir.exists() {
249                    clean_untracked(&dest_dir)?;
250                } else {
251                    std::fs::create_dir_all(&dest_dir)
252                        .with_context(|| format!("failed to create {}", dest_dir.display()))?;
253                }
254
255                http::download(&url, &tarball)
256                    .with_context(|| format!("failed to download {name}"))?;
257
258                fs::extract_tarball_strip_one(&tarball, &dest_dir)
259                    .with_context(|| format!("failed to extract {name}"))?;
260
261                std::fs::remove_file(&tarball).ok();
262
263                write_marker(&dest_dir, SOURCE_READY_MARKER, name, "source")?;
264            }
265            LibrarySource::Git { repo, tag } => {
266                if !dest_dir.exists() {
267                    std::fs::create_dir_all(&dest_dir)
268                        .with_context(|| format!("failed to create {}", dest_dir.display()))?;
269                }
270
271                git_clone_tag(&repo, &tag, &dest_dir)
272                    .with_context(|| format!("failed to clone {name}"))?;
273
274                let autogen = dest_dir.join("autogen.sh");
275                if autogen.exists() {
276                    cmd::run("./autogen.sh", &[], &dest_dir, &[])
277                        .with_context(|| format!("failed to run autogen.sh for {name}"))?;
278                }
279
280                write_marker(&dest_dir, SOURCE_READY_MARKER, name, "source")?;
281            }
282        }
283    }
284
285    Ok(())
286}
287
288/// Clones `repo` at `tag` into `dest`, recursing into submodules.
289///
290/// Clones into a temporary sibling directory first, then moves the contents
291/// into `dest`.  This preserves any files already in `dest` that are tracked
292/// in the cadmus repository (e.g. `build-kobo.sh`), matching the behaviour of
293/// tarball extraction with `strip_one`.
294fn git_clone_tag(repo: &str, tag: &str, dest: &Path) -> Result<()> {
295    let tmp = dest.with_extension("_clone_tmp");
296
297    if tmp.exists() {
298        std::fs::remove_dir_all(&tmp)
299            .with_context(|| format!("failed to remove {}", tmp.display()))?;
300    }
301
302    cmd::run(
303        "git",
304        &[
305            "clone",
306            "--depth=1",
307            "--recurse-submodules",
308            "--branch",
309            tag,
310            repo,
311            tmp.to_str().context("tmp path is not valid UTF-8")?,
312        ],
313        std::path::Path::new("."),
314        &[],
315    )?;
316
317    for entry in
318        std::fs::read_dir(&tmp).with_context(|| format!("failed to read {}", tmp.display()))?
319    {
320        let entry = entry.with_context(|| format!("failed to read entry in {}", tmp.display()))?;
321
322        if entry.file_name() == ".git" {
323            continue;
324        }
325
326        let target = dest.join(entry.file_name());
327        if target.exists() {
328            if target.is_dir() {
329                std::fs::remove_dir_all(&target).ok();
330            } else {
331                std::fs::remove_file(&target).ok();
332            }
333        }
334        std::fs::rename(entry.path(), &target).with_context(|| {
335            format!(
336                "failed to move {} to {}",
337                entry.path().display(),
338                target.display()
339            )
340        })?;
341    }
342
343    std::fs::remove_dir_all(&tmp).with_context(|| format!("failed to remove {}", tmp.display()))?;
344
345    Ok(())
346}
347
348/// Sentinel file written inside a library directory after source extraction.
349///
350/// Its presence means the source tree was fetched and unpacked successfully.
351pub const SOURCE_READY_MARKER: &str = ".source-ready";
352
353/// Sentinel file written inside a library directory after a successful build.
354///
355/// Its presence means the library was already compiled and cached — both the
356/// patch and the build step can be skipped on the next run.
357pub const BUILT_MARKER: &str = ".built-kobo";
358
359/// Returns `true` if `dir` already has a completed source download marker.
360fn is_source_ready(dir: &Path) -> bool {
361    dir.join(SOURCE_READY_MARKER).exists()
362}
363
364/// Returns `true` if the library in `dir` was already built and the sentinel
365/// file is present.
366fn is_built(dir: &Path) -> bool {
367    dir.join(BUILT_MARKER).exists()
368}
369
370/// Writes a marker file inside `dir` to persist task completion state.
371fn write_marker(dir: &Path, marker: &str, name: &str, state: &str) -> Result<()> {
372    std::fs::write(dir.join(marker), "")
373        .with_context(|| format!("failed to write {state} marker for {name}"))
374}
375
376/// Builds the given libraries for the Kobo ARM target.
377///
378/// When `names` is empty all libraries are built in dependency order.  For
379/// each library, `kobo.patch` is applied if present, then `./build-kobo.sh`
380/// is invoked.  A sentinel file ([`BUILT_MARKER`]) is written on success so
381/// that a warm CI cache can skip already-built libraries without re-applying
382/// the patch or re-running the build script.
383///
384/// # Errors
385///
386/// Returns an error if patching or building any library fails.
387pub fn build_libraries(thirdparty_dir: &Path, names: &[&str]) -> Result<()> {
388    let targets: Vec<&str> = if names.is_empty() {
389        LIBRARY_NAMES.to_vec()
390    } else {
391        names.to_vec()
392    };
393
394    for name in targets {
395        let lib_dir = thirdparty_dir.join(name);
396
397        if !lib_dir.exists() {
398            bail!(
399                "thirdparty/{name} not found — run `cargo xtask build-kobo --download-only` first"
400            );
401        }
402
403        if is_built(&lib_dir) {
404            println!("Skipping {name} (already built)…");
405            continue;
406        }
407
408        println!("Building {name}…");
409
410        let patch = lib_dir.join("kobo.patch");
411        if patch.exists() {
412            cmd::run("patch", &["-p", "1", "-i", "kobo.patch"], &lib_dir, &[])
413                .with_context(|| format!("failed to apply kobo.patch for {name}"))?;
414        }
415
416        let envs = [
417            ("AR", "arm-linux-gnueabihf-ar"),
418            ("AS", "arm-linux-gnueabihf-as"),
419            ("STRIP", "arm-linux-gnueabihf-strip"),
420            ("RANLIB", "arm-linux-gnueabihf-ranlib"),
421            ("LD", "arm-linux-gnueabihf-ld"),
422            ("CC_FOR_BUILD", "cc"),
423            ("CXX_FOR_BUILD", "c++"),
424            ("CC_BUILD", "cc"),
425        ];
426        cmd::run("./build-kobo.sh", &[], &lib_dir, &envs)
427            .with_context(|| format!("failed to build {name}"))?;
428
429        write_marker(&lib_dir, BUILT_MARKER, name, "build")?;
430        write_marker(&lib_dir, SOURCE_READY_MARKER, name, "source")?;
431    }
432
433    Ok(())
434}
435
436/// Removes untracked files from a directory using `git ls-files`, falling back
437/// to removing and recreating the directory when git is unavailable.
438pub fn clean_untracked(dir: &Path) -> Result<()> {
439    let result = std::process::Command::new("git")
440        .args(["ls-files", "-o", "--directory", "-z"])
441        .arg(dir.file_name().unwrap_or(dir.as_os_str()))
442        .current_dir(dir.parent().unwrap_or(dir))
443        .output();
444
445    match result {
446        Ok(output) if output.status.success() => {
447            for entry in output.stdout.split(|&b| b == 0) {
448                if entry.is_empty() {
449                    continue;
450                }
451
452                let path = dir
453                    .parent()
454                    .unwrap_or(dir)
455                    .join(std::str::from_utf8(entry).unwrap_or(""));
456
457                if path.is_dir() {
458                    std::fs::remove_dir_all(&path).ok();
459                } else {
460                    std::fs::remove_file(&path).ok();
461                }
462            }
463        }
464        _ => {
465            std::fs::remove_dir_all(dir)
466                .with_context(|| format!("failed to remove {}", dir.display()))?;
467            std::fs::create_dir_all(dir)
468                .with_context(|| format!("failed to recreate {}", dir.display()))?;
469        }
470    }
471
472    Ok(())
473}
474
475#[cfg(test)]
476mod tests {
477    use super::*;
478
479    #[test]
480    fn library_source_is_defined_for_all_known_libraries() {
481        for name in LIBRARY_NAMES {
482            let source = library_source(name).unwrap();
483            match source {
484                LibrarySource::Tarball(url) => {
485                    assert!(
486                        url.starts_with("http"),
487                        "tarball URL for {name} should start with http"
488                    );
489                    assert!(
490                        url.contains(".tar.gz"),
491                        "tarball URL for {name} should contain .tar.gz"
492                    );
493                }
494                LibrarySource::Git { repo, tag } => {
495                    assert!(
496                        repo.starts_with("https://"),
497                        "git repo for {name} should use https"
498                    );
499                    assert!(!tag.is_empty(), "git tag for {name} should not be empty");
500                }
501            }
502        }
503    }
504
505    #[test]
506    fn library_source_errors_on_unknown_library() {
507        assert!(library_source("nonexistent").is_err());
508    }
509
510    #[test]
511    fn library_names_has_no_duplicates() {
512        let mut names = LIBRARY_NAMES.to_vec();
513        names.sort_unstable();
514        names.dedup();
515        assert_eq!(
516            names.len(),
517            LIBRARY_NAMES.len(),
518            "duplicate library names found"
519        );
520    }
521}
xtask_lib/tasks/util/thirdparty.rs

xtask_lib/tasks/util/
thirdparty.rs
