1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
|
// TODO: Add signatures checking and fix mixed sync/async use.
use crate::cfg::config::Config;
use crate::pkgtoolkit::pkgtools::Package;
use flate2::read::GzDecoder;
use futures_util::stream::TryStreamExt;
use indicatif::{ProgressBar, ProgressStyle};
use reqwest;
use serde::Deserialize;
use std::fs::File as StdFile;
use std::{collections::HashMap, path::Path};
use tar::Archive;
use tokio::{fs::File, io::AsyncWriteExt};
pub struct HTTPPackage {
pub config: Config,
pub index_packages: Option<HashMap<String, Package>>,
}
#[derive(Deserialize, Debug)]
struct IndexData {
packages: Vec<Package>,
}
impl HTTPPackage {
/// Creates a new HTTPPackage object with the given configuration.
///
/// # Returns
///
/// A new HTTPPackage object with the given configuration.
pub fn new(config: Config) -> Self {
HTTPPackage {
config,
index_packages: None,
}
}
/// Downloads the INDEX.tar.gz file from the configured repository
/// and stores it in the configured cache directory with a progress bar.
///
/// # Errors
///
/// Returns an error if the request fails, if the response status is not successful, or if there's an issue while reading or writing the file.
pub async fn fetch_index_http(&mut self) -> Result<bool, Box<dyn std::error::Error>> {
let repo_url_opt = &self.config.repo.repo_http_url;
let cache_dir = &self.config.paths.cache_dir;
log::debug!("Cache directory: {:?}", cache_dir);
let index_url = repo_url_opt.as_deref().map_or_else(
|| {
log::warn!(
"Repository URL is None, please specify it in /etc/mesk/mesk.toml and restart."
);
String::new()
},
|repo_url_str| {
if repo_url_str.ends_with(".tar.gz") {
repo_url_str.to_string()
} else {
format!("{}/INDEX.tar.gz", repo_url_str.trim_end_matches('/'))
}
},
);
let client = reqwest::Client::new();
// Make a HEAD request to get the content length for the progress bar
let head_response = client.head(&index_url).send().await?;
let content_length: u64 = head_response
.headers()
.get(reqwest::header::CONTENT_LENGTH)
.and_then(|ct_len| ct_len.to_str().ok())
.and_then(|ct_len| ct_len.parse().ok())
.unwrap_or(0);
// Create progress bar
let pb = if content_length > 0 {
let pb = ProgressBar::new(content_length);
pb.set_style(
ProgressStyle::default_bar()
.template("{spinner:.green} [{elapsed_precise}] [{bar:40.cyan/blue}] {bytes}/{total_bytes} ({eta})")?
.progress_chars("#>-"),
);
pb
} else {
let pb = ProgressBar::new_spinner();
pb.set_style(
ProgressStyle::default_spinner()
.template("{spinner:.green} [{elapsed_precise}] Fetching INDEX.tar.gz...")?,
);
pb
};
// Send GET request and stream the response body
let response = client.get(&index_url).send().await?;
if !response.status().is_success() {
return Err(format!("HTTP Error: {}", response.status()).into());
}
let mut stream = response.bytes_stream();
let file_path = Path::new(cache_dir).join("INDEX.tar.gz");
// Ensure cache_dir exists
tokio::fs::create_dir_all(cache_dir)
.await
.map_err(|e| format!("Failed to create cache dir: {}", e))?;
let mut file = File::create(&file_path).await?;
let mut downloaded: u64 = 0;
while let Some(chunk) = stream.try_next().await? {
file.write_all(&chunk).await?;
let chunk_len = chunk.len() as u64;
downloaded += chunk_len;
pb.set_position(downloaded);
}
pb.finish_with_message("INDEX.tar.gz download finished");
log::info!("Extracting INDEX.tar.gz to cache directory...");
// Package::extract_archive(&file_path.to_string_lossy())?;
let archive_file =
StdFile::open(&file_path).map_err(|e| format!("Failed to open archive: {}", e))?;
let gz_decoder = GzDecoder::new(archive_file);
let mut archive = Archive::new(gz_decoder);
archive
.unpack(cache_dir)
.map_err(|e| format!("Failed to unpack archive: {}", e))?;
let index_toml_path = Path::new(cache_dir).join("INDEX.toml");
if !index_toml_path.exists() {
log::warn!("INDEX.toml not found in INDEX.tar.gz. Proceeding without index data.");
self.index_packages = Some(HashMap::new());
return Ok(true);
}
let index_content = tokio::fs::read_to_string(&index_toml_path).await?;
log::debug!("Content of INDEX.toml:\n{}", index_content);
let index_data: IndexData = toml::from_str(&index_content)
.map_err(|e| format!("Failed to parse INDEX.toml: {}", e))?;
let mut package_map = HashMap::new();
for pkg in index_data.packages {
let base_url = url::Url::parse(&self.config.repo.repo_url)?;
let full_url = base_url.join(&pkg.url)?;
let mut pkg_clone = pkg.clone();
pkg_clone.url = full_url.to_string();
package_map.insert(pkg_clone.name.clone(), pkg_clone);
}
self.index_packages = Some(package_map.clone());
log::info!(
"Index loaded successfully, {} packages found.",
package_map.len()
);
Ok(true)
}
/// An internal auxiliary function for downloading data and writing it to a file with a progress display.
///
/// # Arguments
/// * `url` - The URL to download from.
/// * `file_path` - The path to the file to write the data to.
/// * `description` - Description of the operation for the progress bar.
///
/// # Errors
///
/// Returns an error if the request fails, if the response status is not successful, or if there's an issue while writing the file.
async fn download_file_with_progress(
client: &reqwest::Client,
url: &str,
file_path: &Path,
description: &str,
) -> Result<(), Box<dyn std::error::Error>> {
// Make a HEAD request to get the content length for the progress bar
let head_response = client.head(url).send().await?;
let content_length: u64 = head_response
.headers()
.get(reqwest::header::CONTENT_LENGTH)
.and_then(|ct_len| ct_len.to_str().ok())
.and_then(|ct_len| ct_len.parse().ok())
.unwrap_or(0);
let pb = if content_length > 0 {
let pb = ProgressBar::new(content_length);
pb.set_style(
ProgressStyle::default_bar()
.template(&format!(
"{{spinner:.green}} [{{elapsed_precise}}] [{{bar:40.cyan/blue}}] {{bytes}}/{{total_bytes}} ({} {{eta}})",
description
))?
.progress_chars("#>-"),
);
pb
} else {
let pb = ProgressBar::new_spinner();
pb.set_style(ProgressStyle::default_spinner().template(&format!(
"{{spinner:.green}} [{{elapsed_precise}}] {}...",
description
))?);
pb
};
// Send GET request and stream the response body
let response = client.get(url).send().await?;
if !response.status().is_success() {
return Err(format!("HTTP Error: {}", response.status()).into());
}
let mut stream = response.bytes_stream();
let mut file = File::create(&file_path).await?;
let mut downloaded: u64 = 0;
while let Some(chunk) = stream.try_next().await? {
file.write_all(&chunk).await?;
let chunk_len = chunk.len() as u64;
downloaded += chunk_len;
pb.set_position(downloaded);
}
pb.finish_with_message(format!("{} download finished", description));
Ok(())
}
/// Fetches a specific package identified by `package_name`.
/// Assumes `fetch_index_http` has been called and `self.index_packages` is populated to get the URL.
/// Downloads the package file (.mesk) to the cache directory with a progress bar.
///
/// # Errors
///
/// Returns an error if the index is not loaded, the package is not found in the index,
/// the package URL is invalid, the request fails, or if there's an issue writing the file.
pub async fn fetch_package_http(
&self,
package_name: &str,
) -> Result<bool, Box<dyn std::error::Error>> {
let package_info = self.fetch_package_info(package_name)?;
let url = &package_info.url;
let client = reqwest::Client::new();
let file_name = Path::new(url)
.file_name()
.ok_or("Could not determine filename from URL")?
.to_str()
.ok_or("Filename is not valid UTF-8")?;
let cache_dir = &self.config.paths.cache_dir;
let file_path = Path::new(cache_dir).join(file_name);
tokio::fs::create_dir_all(&cache_dir).await?;
Self::download_file_with_progress(&client, url, &file_path, file_name).await?;
log::info!(
"Package '{}' downloaded successfully to {:?}",
package_name,
file_path
);
Ok(true)
}
/// Fetches a specific package identified by `index` (likely the package name).
/// Assumes `fetch_index_http` has been called and `self.index_packages` is populated.
pub fn fetch_package_info(
&self,
package_name: &str,
) -> Result<&Package, Box<dyn std::error::Error>> {
let packages = self
.index_packages
.as_ref()
.ok_or("Index not loaded. Call fetch_index_http first.")?;
let pkg_info = packages
.get(package_name)
.ok_or(format!("Package '{}' not found in index.", package_name))?;
Ok(pkg_info)
}
}
|