Commit 10c28f8a authored by Lorenzo "Palinuro" Faletra's avatar Lorenzo "Palinuro" Faletra
Browse files

Import Debian version 2.1.20+parrot1

apt (2.1.20+parrot1) rolling-testing; urgency=medium
.
  * Import new Debian release.
  * Patch methods/basehttp.c to allow https-to-http redirects.
.
apt (2.1.20) unstable; urgency=medium
.
  * CI: Run test as user on i386
  * Fix test suite regression from StrToNum fixes. The tests started failing
    on 32-bit because the values were actually out of range, but we did not
    test errno before the last version, so it was not treated as an error.
.
apt (2.1.19) unstable; urgency=medium
.
  [ Helge Kreutzmann ]
  * German program translation update (Closes: #979848)
.
  [ Youfu Zhang ]
  * dpkg: fix passing readonly /dev/null fd as stdout/stderr
.
  [ Diederik de Haas ]
  * Fix apt-acquire-additional-files entity's location.
.
  [ Wolfgang Schweer ]
  * vendor: Adjust Debian -security codename
.
  [ Julian Andres Klode ]
  * Include all translations when building the cache (LP: #1907850)
.
  [ David Kalnischkies ]
  * Various patches uplifted from unfinished fuzzer bra...
parent d8fe2d80
Pipeline #2056 failed with stages
in 0 seconds
......@@ -23,6 +23,7 @@ test as root:
- unbuffer ./test/integration/run-tests -q -j 4
test as user:
image: i386/debian:unstable
stage: test
script:
- adduser --home /home/travis travis --quiet --disabled-login --gecos "" --uid 1000
......
# CMake support for target-based function multiversioning
#
# Copyright (C) 2019 Canonical Ltd
#
# Author: Julian Andres Klode <jak@debian.org>.
#
# Permission is hereby granted, free of charge, to any person
# obtaining a copy of this software and associated documentation files
# (the "Software"), to deal in the Software without restriction,
# including without limitation the rights to use, copy, modify, merge,
# publish, distribute, sublicense, and/or sell copies of the Software,
# and to permit persons to whom the Software is furnished to do so,
# subject to the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
# BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
# ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
include(CheckCXXSourceCompiles)
function(check_cxx_target var target code)
check_cxx_source_compiles(
"
__attribute__((target(\"${target}\"))) static int foo(int i) { return ${code}; }
__attribute__((target(\"default\"))) static int foo(int i) { return i; }
int main(int i, char **) { return foo(i); }
" ${var})
endfunction()
......@@ -80,9 +80,5 @@
/* Group of the root user */
#cmakedefine ROOT_GROUP "${ROOT_GROUP}"
/* defined if __builtin_ia32_crc32{s,d}i() exists in an sse4.2 target */
#cmakedefine HAVE_FMV_SSE42_AND_CRC32
#cmakedefine HAVE_FMV_SSE42_AND_CRC32DI
/* unrolling is faster combined with an optimizing compiler */
#define SHA2_UNROLL_TRANSFORM
......@@ -197,15 +197,10 @@ else()
set(RESOLV_LIBRARIES -lresolv)
endif()
# Check multiversioning
include(CheckCxxTarget)
check_cxx_target(HAVE_FMV_SSE42_AND_CRC32 "sse4.2" "__builtin_ia32_crc32si(0,i)|__builtin_ia32_crc32hi(0,i)|__builtin_ia32_crc32qi(0,i)")
check_cxx_target(HAVE_FMV_SSE42_AND_CRC32DI "sse4.2" "__builtin_ia32_crc32di(0,i)")
# Configure some variables like package, version and architecture.
set(PACKAGE ${PROJECT_NAME})
set(PACKAGE_MAIL "APT Development Team <deity@lists.debian.org>")
set(PACKAGE_VERSION "2.1.18")
set(PACKAGE_VERSION "2.1.20")
string(REGEX MATCH "^[0-9.]+" PROJECT_VERSION ${PACKAGE_VERSION})
if (NOT DEFINED DPKG_DATADIR)
......
......@@ -2604,14 +2604,32 @@ bool pkgAcqDiffIndex::ParseDiffIndex(string const &IndexDiffFile) /*{{{*/
return false;
}
/* decide if we should download patches one by one or in one go:
The first is good if the server merges patches, but many don't so client
based merging can be attempt in which case the second is better.
"bad things" will happen if patches are merged on the server,
but client side merging is attempt as well */
pdiff_merge = _config->FindB("Acquire::PDiffs::Merge", true);
if (pdiff_merge == true)
{
// reprepro and dak add this flag if they merge patches on the server
std::string const precedence = Tags.FindS("X-Patch-Precedence");
pdiff_merge = (precedence != "merged");
}
// calculate the size of all patches we have to get
unsigned short const sizeLimitPercent = _config->FindI("Acquire::PDiffs::SizeLimit", 100);
if (sizeLimitPercent > 0)
{
unsigned long long downloadSize = std::accumulate(available_patches.begin(),
available_patches.end(), 0llu, [](unsigned long long const T, DiffInfo const &I) {
return T + I.download_hashes.FileSize();
});
unsigned long long downloadSize = 0;
if (pdiff_merge)
downloadSize = std::accumulate(available_patches.begin(), available_patches.end(), 0llu,
[](unsigned long long const T, DiffInfo const &I) {
return T + I.download_hashes.FileSize();
});
// if server-side merging, assume we will need only the first patch
else if (not available_patches.empty())
downloadSize = available_patches.front().download_hashes.FileSize();
if (downloadSize != 0)
{
unsigned long long downloadSizeIdx = 0;
......@@ -2636,19 +2654,6 @@ bool pkgAcqDiffIndex::ParseDiffIndex(string const &IndexDiffFile) /*{{{*/
}
}
/* decide if we should download patches one by one or in one go:
The first is good if the server merges patches, but many don't so client
based merging can be attempt in which case the second is better.
"bad things" will happen if patches are merged on the server,
but client side merging is attempt as well */
pdiff_merge = _config->FindB("Acquire::PDiffs::Merge", true);
if (pdiff_merge == true)
{
// reprepro adds this flag if it has merged patches on the server
std::string const precedence = Tags.FindS("X-Patch-Precedence");
pdiff_merge = (precedence != "merged");
}
// clean the plate
{
std::string const Final = GetExistingFilename(CurrentPackagesFile);
......
......@@ -482,9 +482,25 @@ void pkgAcqMethod::PrintStatus(char const * const header, const char* Format,
void pkgAcqMethod::Log(const char *Format,...)
{
va_list args;
va_start(args,Format);
PrintStatus("101 Log", Format, args);
va_end(args);
ssize_t size = 400;
std::ostringstream outstr;
while (true) {
bool ret;
va_start(args,Format);
ret = iovprintf(outstr, Format, args, size);
va_end(args);
if (ret == true)
break;
}
std::unordered_map<std::string, std::string> fields;
if (Queue != 0)
try_emplace(fields, "URI", Queue->Uri);
else
try_emplace(fields, "URI", "<UNKNOWN>");
if (not UsedMirror.empty())
try_emplace(fields, "UsedMirror", UsedMirror);
try_emplace(fields, "Message", outstr.str());
SendMessage("101 Log", std::move(fields));
}
/*}}}*/
// AcqMethod::Status - Send a status message /*{{{*/
......@@ -493,9 +509,25 @@ void pkgAcqMethod::Log(const char *Format,...)
void pkgAcqMethod::Status(const char *Format,...)
{
va_list args;
va_start(args,Format);
PrintStatus("102 Status", Format, args);
va_end(args);
ssize_t size = 400;
std::ostringstream outstr;
while (true) {
bool ret;
va_start(args,Format);
ret = iovprintf(outstr, Format, args, size);
va_end(args);
if (ret == true)
break;
}
std::unordered_map<std::string, std::string> fields;
if (Queue != 0)
try_emplace(fields, "URI", Queue->Uri);
else
try_emplace(fields, "URI", "<UNKNOWN>");
if (not UsedMirror.empty())
try_emplace(fields, "UsedMirror", UsedMirror);
try_emplace(fields, "Message", outstr.str());
SendMessage("102 Status", std::move(fields));
}
/*}}}*/
// AcqMethod::Redirect - Send a redirect message /*{{{*/
......
......@@ -101,7 +101,7 @@ class APT_PUBLIC pkgAcqMethod
bool MediaFail(std::string Required,std::string Drive);
virtual void Exit() {};
void PrintStatus(char const * const header, const char* Format, va_list &args) const;
APT_DEPRECATED_MSG("Use SendMessage instead") void PrintStatus(char const * const header, const char* Format, va_list &args) const;
public:
enum CnfFlags
......
......@@ -27,10 +27,13 @@
#include <apt-pkg/progress.h>
#include <apt-pkg/sourcelist.h>
#include <limits>
#include <memory>
#include <string>
#include <vector>
#include <string.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <unistd.h>
#include <apti18n.h>
......@@ -288,15 +291,27 @@ bool pkgCacheFile::AddIndexFile(pkgIndexFile * const File) /*{{{*/
// CacheFile::RemoveCaches - remove all cache files from disk /*{{{*/
// ---------------------------------------------------------------------
/* */
static void SetCacheStartBeforeRemovingCache(std::string const &cache)
{
if (cache.empty())
return;
auto const CacheStart = _config->FindI("APT::Cache-Start", 0);
constexpr auto CacheStartDefault = 24 * 1024 * 1024;
struct stat Buf;
if (stat(cache.c_str(), &Buf) == 0 && (Buf.st_mode & S_IFREG) != 0)
{
RemoveFile("RemoveCaches", cache);
if (CacheStart == 0 && std::numeric_limits<decltype(CacheStart)>::max() >= Buf.st_size && Buf.st_size > CacheStartDefault)
_config->Set("APT::Cache-Start", Buf.st_size);
}
}
void pkgCacheFile::RemoveCaches()
{
std::string const pkgcache = _config->FindFile("Dir::cache::pkgcache");
SetCacheStartBeforeRemovingCache(pkgcache);
std::string const srcpkgcache = _config->FindFile("Dir::cache::srcpkgcache");
SetCacheStartBeforeRemovingCache(srcpkgcache);
if (pkgcache.empty() == false && RealFileExists(pkgcache) == true)
RemoveFile("RemoveCaches", pkgcache);
if (srcpkgcache.empty() == false && RealFileExists(srcpkgcache) == true)
RemoveFile("RemoveCaches", srcpkgcache);
if (pkgcache.empty() == false)
{
std::string cachedir = flNotFile(pkgcache);
......
......@@ -82,10 +82,10 @@ template<typename Str, typename Itr> class APT_PUBLIC pkgCache::Iterator :
inline unsigned long Index() const {return S - OwnerPointer();}
inline map_pointer<Str> MapPointer() const {return map_pointer<Str>(Index()) ;}
void ReMap(void const * const oldMap, void const * const newMap) {
void ReMap(void const * const oldMap, void * const newMap) {
if (Owner == 0 || S == 0)
return;
S += static_cast<Str const *>(newMap) - static_cast<Str const *>(oldMap);
S = static_cast<Str *>(newMap) + (S - static_cast<Str const *>(oldMap));
}
// Constructors - look out for the variable assigning
......@@ -350,12 +350,12 @@ class APT_PUBLIC pkgCache::DepIterator : public Iterator<Dependency, DepIterator
};
inline DependencyProxy operator->() const {return (DependencyProxy) { S2->Version, S2->Package, S->ID, S2->Type, S2->CompareOp, S->ParentVer, S->DependencyData, S->NextRevDepends, S->NextDepends, S2->NextData };}
inline DependencyProxy operator->() {return (DependencyProxy) { S2->Version, S2->Package, S->ID, S2->Type, S2->CompareOp, S->ParentVer, S->DependencyData, S->NextRevDepends, S->NextDepends, S2->NextData };}
void ReMap(void const * const oldMap, void const * const newMap)
void ReMap(void const * const oldMap, void * const newMap)
{
Iterator<Dependency, DepIterator>::ReMap(oldMap, newMap);
if (Owner == 0 || S == 0 || S2 == 0)
return;
S2 += static_cast<DependencyData const *>(newMap) - static_cast<DependencyData const *>(oldMap);
S2 = static_cast<DependencyData *>(newMap) + (S2 - static_cast<DependencyData const *>(oldMap));
}
//Nice printable representation
......
......@@ -464,14 +464,12 @@ bool pkgCdrom::WriteSourceList(string Name,vector<string> &List,bool Source)
string File = _config->FindFile("Dir::Etc::sourcelist");
// Open the stream for reading
ifstream F((FileExists(File)?File.c_str():"/dev/null"),
ios::in );
if (F.fail() == true)
return _error->Errno("ifstream::ifstream","Opening %s",File.c_str());
FileFd F(FileExists(File) ? File : "/dev/null", FileFd::ReadOnly);
if (not F.IsOpen() || F.Failed())
return _error->Errno("WriteSourceList", "Opening %s failed", File.c_str());
string NewFile = File + ".new";
RemoveFile("WriteDatabase", NewFile);
RemoveFile("WriteSourceList", NewFile);
ofstream Out(NewFile.c_str());
if (!Out)
return _error->Errno("ofstream::ofstream",
......@@ -487,21 +485,16 @@ bool pkgCdrom::WriteSourceList(string Name,vector<string> &List,bool Source)
else
Type = "deb";
char Buffer[300];
int CurLine = 0;
bool First = true;
while (F.eof() == false)
{
F.getline(Buffer,sizeof(Buffer));
CurLine++;
if (F.fail() && !F.eof())
return _error->Error(_("Line %u too long in source list %s."),
CurLine,File.c_str());
_strtabexpand(Buffer,sizeof(Buffer));
_strstrip(Buffer);
std::string Buffer;
while (F.ReadLine(Buffer))
{
++CurLine;
auto const Cleaned = APT::String::Strip(SubstVar(Buffer, "\t", " "));
// Comment or blank
if (Buffer[0] == '#' || Buffer[0] == 0)
if (Cleaned.empty() || Cleaned[0] == '#')
{
Out << Buffer << endl;
continue;
......@@ -523,7 +516,7 @@ bool pkgCdrom::WriteSourceList(string Name,vector<string> &List,bool Source)
// Grok it
string cType;
string URI;
const char *C = Buffer;
const char *C = Cleaned.c_str();
if (ParseQuoteWord(C,cType) == false ||
ParseQuoteWord(C,URI) == false)
{
......
......@@ -21,6 +21,7 @@
#include <apt-pkg/fileutl.h>
#include <apt-pkg/macros.h>
#include <apt-pkg/strutl.h>
#include <apt-pkg/string_view.h>
#include <ctype.h>
#include <regex.h>
......@@ -861,107 +862,82 @@ bool ReadConfigFile(Configuration &Conf,const string &FName,bool const &AsSectio
// The input line with comments stripped.
std::string Fragment;
// Expand tabs in the input line and remove leading and trailing
// whitespace.
{
const int BufferSize = Input.size() * 8 + 1;
char *Buffer = new char[BufferSize];
try
{
memcpy(Buffer, Input.c_str(), Input.size() + 1);
_strtabexpand(Buffer, BufferSize);
_strstrip(Buffer);
Input = Buffer;
}
catch(...)
{
delete[] Buffer;
throw;
}
delete[] Buffer;
}
// Expand tabs in the input line and remove leading and trailing whitespace.
Input = APT::String::Strip(SubstVar(Input, "\t", " "));
CurLine++;
// Now strip comments; if the whole line is contained in a
// comment, skip this line.
APT::StringView Line{Input.data(), Input.size()};
// The first meaningful character in the current fragment; will
// be adjusted below as we remove bytes from the front.
std::string::const_iterator Start = Input.begin();
// The last meaningful character in the current fragment.
std::string::const_iterator End = Input.end();
// Multi line comment
if (InComment == true)
// continued Multi line comment
if (InComment)
{
for (std::string::const_iterator I = Start;
I != End; ++I)
size_t end = Line.find("*/");
if (end != APT::StringView::npos)
{
if (*I == '*' && I + 1 != End && I[1] == '/')
{
Start = I + 2;
InComment = false;
break;
}
Line.remove_prefix(end + 2);
InComment = false;
}
if (InComment == true)
else
continue;
}
// Discard single line comments
bool InQuote = false;
for (std::string::const_iterator I = Start;
I != End; ++I)
{
if (*I == '"')
InQuote = !InQuote;
if (InQuote == true)
continue;
if ((*I == '/' && I + 1 != End && I[1] == '/') ||
(*I == '#' && strcmp(string(I,I+6).c_str(),"#clear") != 0 &&
strcmp(string(I,I+8).c_str(),"#include") != 0 &&
strcmp(string(I,I+strlen("#x-apt-configure-index")).c_str(), "#x-apt-configure-index") != 0))
size_t start = 0;
while ((start = Line.find("//", start)) != APT::StringView::npos)
{
End = I;
if (std::count(Line.begin(), Line.begin() + start, '"') % 2 != 0)
{
++start;
continue;
}
Line.remove_suffix(Line.length() - start);
break;
}
using APT::operator""_sv;
constexpr std::array<APT::StringView, 3> magicComments { "clear"_sv, "include"_sv, "x-apt-configure-index"_sv };
start = 0;
while ((start = Line.find('#', start)) != APT::StringView::npos)
{
if (std::count(Line.begin(), Line.begin() + start, '"') % 2 != 0 ||
std::any_of(magicComments.begin(), magicComments.end(), [&](auto const m) { return Line.compare(start+1, m.length(), m) == 0; }))
{
++start;
continue;
}
Line.remove_suffix(Line.length() - start);
break;
}
}
// Look for multi line comments and build up the
// fragment.
Fragment.reserve(End - Start);
InQuote = false;
for (std::string::const_iterator I = Start;
I != End; ++I)
Fragment.reserve(Line.length());
{
if (*I == '"')
InQuote = !InQuote;
if (InQuote == true)
Fragment.push_back(*I);
else if (*I == '/' && I + 1 != End && I[1] == '*')
{
InComment = true;
for (std::string::const_iterator J = I;
J != End; ++J)
size_t start = 0;
while ((start = Line.find("/*", start)) != APT::StringView::npos)
{
if (std::count(Line.begin(), Line.begin() + start, '"') % 2 != 0)
{
if (*J == '*' && J + 1 != End && J[1] == '/')
{
// Pretend we just finished walking over the
// comment, and don't add anything to the output
// fragment.
I = J + 1;
InComment = false;
break;
}
start += 2;
continue;
}
if (InComment == true)
break;
Fragment.append(Line.data(), start);
auto const end = Line.find("*/", start + 2);
if (end == APT::StringView::npos)
{
Line.clear();
InComment = true;
break;
}
else
Line.remove_prefix(end + 2);
start = 0;
}
else
Fragment.push_back(*I);
if (not Line.empty())
Fragment.append(Line.data(), Line.length());
}
// Skip blank lines.
......@@ -969,9 +945,9 @@ bool ReadConfigFile(Configuration &Conf,const string &FName,bool const &AsSectio
continue;
// The line has actual content; interpret what it means.
InQuote = false;
Start = Fragment.begin();
End = Fragment.end();
bool InQuote = false;
auto Start = Fragment.cbegin();
auto End = Fragment.cend();
for (std::string::const_iterator I = Start;
I != End; ++I)
{
......@@ -1150,10 +1126,13 @@ bool ReadConfigFile(Configuration &Conf,const string &FName,bool const &AsSectio
bool ReadConfigDir(Configuration &Conf,const string &Dir,
bool const &AsSectional, unsigned const &Depth)
{
_error->PushToStack();
auto const files = GetListOfFilesInDir(Dir, "conf", true, true);
auto const successfulList = not _error->PendingError();
_error->MergeWithStack();
return std::accumulate(files.cbegin(), files.cend(), true, [&](bool good, auto const &file) {
return ReadConfigFile(Conf, file, AsSectional, Depth) && good;
});
}) && successfulList;
}
/*}}}*/
// MatchAgainstConfig Constructor /*{{{*/
......
......@@ -1774,9 +1774,8 @@ public:
#endif
};
/*}}}*/
class APT_HIDDEN ZstdFileFdPrivate : public FileFdPrivate
{ /*{{{*/
class APT_HIDDEN ZstdFileFdPrivate : public FileFdPrivate /*{{{*/
{
#ifdef HAVE_ZSTD
ZSTD_DStream *dctx;
ZSTD_CStream *cctx;
......@@ -1986,7 +1985,7 @@ class APT_HIDDEN ZstdFileFdPrivate : public FileFdPrivate
#endif
};
/*}}}*/
class APT_HIDDEN LzmaFileFdPrivate: public FileFdPrivate { /*{{{*/
class APT_HIDDEN LzmaFileFdPrivate: public FileFdPrivate { /*{{{*/
#ifdef HAVE_LZMA
struct LZMAFILE {
FILE* file;
......@@ -2092,17 +2091,9 @@ public:
}
else
{
uint64_t const memlimit = UINT64_MAX;
if (compressor.Name == "xz")
{
if (lzma_auto_decoder(&lzma->stream, memlimit, 0) != LZMA_OK)
return false;
}
else
{
if (lzma_alone_decoder(&lzma->stream, memlimit) != LZMA_OK)
return false;
}
uint64_t constexpr memlimit = 1024 * 1024 * 500;
if (lzma_auto_decoder(&lzma->stream, memlimit, 0) != LZMA_OK)
return false;
lzma->compressing = false;
}
return true;
......
......@@ -121,21 +121,6 @@ static bool operator!=(LineBuffer const &buf, APT::StringView const exp) noexcep
And as a cherry on the cake, we use our apt-key wrapper to do part
of the lifting in regards to merging keyrings. Fun for the whole family.
*/
static bool iovprintf(std::ostream &out, const char *format,
va_list &args, ssize_t &size) {
auto S = make_unique_char(malloc(size));
ssize_t const n = vsnprintf(S.get(), size, format, args);
if (n > -1 && n < size) {
out << S.get();
return true;
} else {
if (n > -1)
size = n + 1;
else
size *= 2;
}
return false;
}
static void APT_PRINTF(4) apt_error(std::ostream &outterm, int const statusfd, int fd[2], const char *format, ...)
{
std::ostringstream outstr;
......
......@@ -393,7 +393,7 @@ unsigned long DynamicMMap::Allocate(unsigned long ItemSize)
bool const newError = _error->PendingError();
_error->MergeWithStack();
if (Pools != oldPools)
I += Pools - oldPools;
I = Pools + (I - oldPools);