Commit bafda979 authored by Nong Hoang Tu's avatar Nong Hoang Tu
Browse files

Use new source code

parent f07a1b9b
import gbputils
import os
proc doClone*(prjName, srcFolder: string): int =
let pwd = getCurrentDir()
setCurrentDir(srcFolder)
result = gbpClone(prjName)
setCurrentDir(pwd)
import strutils
import os
import logging
import times
type
DebProject* = object
handle*: FileLogger
name*: string # name of project
path*: string # full project path
const
UPTODATED* = 0
OUTOFDATED* = 1
ERR* = -1
COMPLETED* = 2
proc dirName*(name: string): string =
#[
Add slash for folder name
]#
result = if not name.endsWith("/"): name & "/" else: name
proc initLogFolder*(path: string): string =
let
folderName = now().format("dd-MM-yyyy")
result = dirName(path & folderName)
try:
createDir(result)
return result
except:
echo "[x] Error while making log folder"
echo getCurrentExceptionMsg()
return ""
proc initLogResult*(path: string): FileLogger =
return newFileLogger(path & "result.log")
proc initProject*(srcFolder, logFolder, prjName: string): DebProject =
#[
srcFolder: string = Full path of Debian project that we update source code
logFolder: string = Full path of log (full log) that we use to write
prjName: string = Project name of Debian project
]#
let
projectPath = srcFolder & prjName
logger = newFileLogger(logFolder & "fulllog.log")
return DebProject(handle: logger, path: projectPath, name: prjName)
#[
This module is the wrapper of gbp commands
Requires package git-buidpackage
]#
import osproc
type
CmdErr* = enum
success
proc gbpPull*(): int =
#[
Run gbp pull to sync any changes to avoid conflict
It must be run before doing anything on project
]#
return execCmd("gbp pull")
proc gbpUpdate*(): int =
#[
Update debian project with gbp + uscan
]#
return execCmd("gbp import-orig --no-interactive --pristine-tar --uscan")
proc gbpTag*(): int =
#[
Generate git tags for version change
Solved gbp:error: revision 'debian/1.3.0-parrot0^{commit}' not found
]#
return execCmd("gbp tag")
proc gbpChange*(version: string): int =
#[
Generate new Debian version using gbp dch
Distro name will be forced as rolling-testing (flag: --distribution=rolling-testing)
Add flag --force-distribution to avoid stdin requires everytime it runs
Add flag -c to auto commit change of debian/changelog
Add flag -N <version> to solve the problem gbp:error: revision 'debian/1.3.0-parrot0^{commit}' not found
The version name should be added -parrot0 by default. The -parrot0 is hardcoded text as a new upstream release
]#
return execCmd("gbp dch --distribution=rolling-testing --force-distribution -c -N " & version & "-parrot0")
proc gbpPush*(): int =
#[
Push all changes with branches to nest
PROBLEM: on update server, maintainer have to type ssh password multiple times
]#
return execCmd("gbp push --pristine-tar")
proc gbpClone*(packageName: string): int =
return execCmd("gbp clone git@git.parrot.sh:packages/tools/" & packageName)
import core
import timing
import logging
proc makeLogMsg(name, msg: string): string =
return "[" & getLogTime() & "] [" & name & "] " & msg
method logInfo*(obj: DebProject, event: string) {.base.} =
obj.handle.log(lvlInfo, makeLogMsg(obj.name, event))
echo " [+] ", event
method logError*(obj: DebProject, event: string) {.base.} =
obj.handle.log(lvlError, makeLogMsg(obj.name, event))
echo " [x] ", event
method logNoti*(obj: DebProject, event: string) {.base.} =
obj.handle.log(lvlInfo, makeLogMsg(obj.name, event))
let logTime = getLogTime()
echo "[i] [", logtime, "] [", obj.name, "] [", event, "]"
proc retError*(obj: FileLogger, prjName: string) =
# Log for the result
let logTime = getLogTime()
echo "[i] [", logtime, "] [", prjName, "] [Update failed]"
obj.log(lvlError, makeLogMsg(prjName, "Update error"))
proc retLatest*(obj: FileLogger, prjName: string) =
# Log for the result
let logTime = getLogTime()
echo "[i] [", logtime, "] [", prjName, "] [Project is up-to-dated]"
obj.log(lvlInfo, makeLogMsg(prjName, "Update up-to-dated"))
proc retCompleted*(obj: FileLogger, prjName: string) =
# Log for the result
let logTime = getLogTime()
echo "[i] [", logtime, "] [", prjName, "] [Update completed]"
obj.log(lvlInfo, makeLogMsg(prjName, "Update completed"))
import httpclient
import strutils
import log
import logging
import json
type
NewUpstream* = object
version*: string
tarURL*: string
error*: bool
proc parseVersion*(line: string): string =
result = line.split(" ")[1]
result = result.replace("(", "").replace(")", "")
if "+" in result:
return join(result.split("+")[0 .. ^2], "+")
else:
return join(result.split("-")[0 .. ^2], "-")
proc getLocalVersion*(path: string): string =
#[
Parse last version from changelog
changelog is in $path/debian/changelog
]#
let
changelogPath = path & "debian/changelog"
try:
let
f = open(changelogPath, fmRead)
lastLog = f.readLine()
f.close()
result = parseVersion(lastLog)
except:
result = ""
fullLog.log(lvlFatal, makeLogMsg(changelogPath, "Can not get Debian version"))
fullLog.log(lvlDebug, makeLogMsg(changelogPath, getCurrentExceptionMsg()))
echo getCurrentExceptionMsg()
# proc validVersion*(ver: string): string =
# #[
# Server can have bad char like v3.5.6,
# We get only int and dot
# ]#
# # FIXME: mimikatz (1:2.2.0-20200229-1parrot2)
# return ver
proc getGitData*(repo: string): NewUpstream =
#[
Get version from github api
Syntax: https://api.github.com/repos/$1/releases/latest
Example: https://api.github.com/repos/radareorg/radare2/releases/latest
radareorg/radare2 -> $1
Update: Add get version and tarbal from tag name
Example: https://api.github.com/repos/rapid7/metasploit-framework/tags
Source: https://gist.github.com/lukechilds/a83e1d7127b78fef38c2914c4ececc3c
]#
# if not url.endsWith("/")
var
client = newHttpClient()
thisUpstream = NewUpstream()
try:
let
releaseAPI = "https://api.github.com/repos/" & repo & "/releases/latest"
# tagsAPI = "https://api.github.com/repos/" & repo & "/tags"
let resp = client.get(releaseAPI)
if resp.status != "404 Not Found":
# thisUpstream.version = validVersion(parseJson(resp.body)["tag_name"].getStr())
thisUpstream.version = parseJson(resp.body)["tag_name"].getStr()
thisUpstream.tarURL = parseJson(resp.body)["tarball_url"].getStr()
thisUpstream.error = false
else:
thisUpstream.error = true
# let resp = client.get(tagsAPI)
# if resp.status == "404 Not Found":
# thisUpstream.error = true
# else:
# thisUpstream.version = validVersion(parseJson(resp.body)[0]["name"].getStr())
# thisUpstream.tarURL = parseJson(resp.body)[0]["tarball_url"].getStr()
except:
echo "[x] Error while getting version from server"
fullLog.log(lvlFatal, makeLogMsg(repo, "Can not get Git version"))
fullLog.log(lvlDebug, makeLogMsg(repo, getCurrentExceptionMsg()))
echo getCurrentExceptionMsg()
thisUpstream.error = true
finally:
client.close()
return thisUPstream
import times
import strutils
proc getLogTime*(): string =
let thisTime = now()
return thisTime.format("hh:mm:ss dd-MM-yyyy")
proc getChangelogTime*(): string =
let
thisTime = now()
day = thisTime.format("ddd, dd MMM yyyy hh:mm:ss")
timeZone = thisTime.format("zzz")
return day & " " & timeZone.replace(":", "")
import core
import gbputils
import logutils
method srcPull*(obj: DebProject): bool {.base.} =
obj.logInfo("Pulling latest changes from nest")
let cmdResult = gbpPull()
if cmdResult != 0:
obj.logError("Error while pulling")
return false
else:
obj.logInfo("Pulled from git succesfully")
return true
method srcUpdate*(obj: DebProject): bool {.base.} =
obj.logInfo("Updating source code")
let cmdResult = gbpUpdate()
if cmdResult != 0:
obj.logError("Error while using gbp uscan")
return false
else:
obj.logInfo("Uscan updated succesfully")
return true
method genTag*(obj: DebProject): bool {.base.} =
obj.logInfo("Generating tag")
let cmdResult = gbpTag()
if cmdResult != 0:
obj.logError("Error while generating tag")
return false
else:
obj.logInfo("Created tag using gbp")
return true
method genChangelog*(obj: DebProject, version: string): bool {.base.} =
obj.logInfo("Generating changelog")
let cmdResult = gbpChange(version)
if cmdResult != 0:
obj.logError("Error while generating changelog")
return false
else:
obj.logInfo("Created changelog using gbp")
return true
method srcPush*(obj: DebProject): bool {.base.} =
obj.logInfo("Pushing latest source")
let cmdResult = gbpPush()
if cmdResult != 0:
obj.logError("Error while pushsing source with gbp")
return false
else:
obj.logInfo("Pushed source using gbp")
return true
import httpclient
import os
import timming
import log
import logging
import strutils
import re
import checkVersion
proc upgradeByUscan*(path: string): bool =
# Fixed fatal: A branch named 'upstream' already exists.
#[
Check if ".git/refs/heads/upstream" exists. If not, create upstream branch
]#
setCurrentDir(path)
discard execShellCmd("git pull")
if not fileExists(path & "/.git/refs/heads/upstream"):
let
# Change to upstream branch
cmdSwitchBranch = "git checkout -b upstream"
execSwitchBranch = execShellCmd(cmdSwitchBranch)
if execSwitchBranch != 0:
return false
let
cmd = "gbp import-orig --no-interactive --uscan"
cmdResult = execShellCmd(cmd)
if cmdResult != 0:
return false
return true
proc getLatestTarbar*(url, folder, name, version: string): string =
#[
Download latest source code from github and save to root folder
Return name of full path of tarbar file or empty if has error
]#
var client = newHttpClient()
try:
let saveName = folder & name & "_" & version & ".tar.gz"
fullLog.log(lvlInfo, makeLogMsg(url, "Start downloading"))
client.downloadFile(url, saveName)
fullLog.log(lvlInfo, makeLogMsg(saveName, "Download completed"))
return saveName
except:
fullLog.log(lvlFatal, makeLogMsg(url, "Error while downloading"))
echo "[x] Error while getting version from server"
echo getCurrentExceptionMsg()
fullLog.log(lvlDebug, makeLogMsg(url, getCurrentExceptionMsg()))
return ""
finally:
client.close()
proc updateSource*(tarLocation, gitFolder, version: string): bool =
#[
Use gbp (git-buildpackage) to update following debian standard
command format:
1. Go to git project folder
2. gbp import-orig <source code folder> -u <version>
]#
try:
setCurrentDir(gitFolder)
discard execShellCmd("git pull")
# let
# # Change to upstream branch
# cmdMakeUpstream = "git checkout -b upstream"
# cmdSwitchBranch = "git checkout upstream"
# execSwitchBranch = execShellCmd(cmdSwitchBranch)
# if execSwitchBranch != 0:
# let execMakeUpstream = execShellCmd(cmdMakeUpstream)
# if execMakeUpstream != 0:
# return false
if not fileExists(gitFolder & "/.git/refs/heads/upstream"):
let
# Change to upstream branch
cmdSwitchBranch = "git checkout -b upstream"
execSwitchBranch = execShellCmd(cmdSwitchBranch)
if execSwitchBranch != 0:
return false
let
# Import latest version
cmdUpdateSource = "gbp import-orig " & tarLocation & " -u " & version
execUpdateSource = execShellCmd(cmdUpdateSource)
if execUpdateSource == 0:
return true
return false
except:
return false
proc upgradeByWatchFile*(path: string): NewUpstream =
#[
Read the <path>/debian/watch
Simulate the uscan by parsing http
]#
var thisUpstream = NewUpstream()
for line in lines(path & "debian/watch"):
if line.startsWith("http"):
let
url = line.split(" ")[0]
# regex = line.split(" ")[1]
var client = newHttpClient()
try:
let resp = client.get(url)
if resp.status == "404 Not Found":
thisUpstream.error = true
else:
let lastTarball = findAll(resp.body, re("[a-zA-Z0-9\\/\\-\\.]+\\/archive\\/[a-zA-Z0-9\\/\\-\\.]+\\.tar\\.gz"))[0]
thisUpstream.tarURL = "https://github.com" & lastTarball
thisUpstream.version = splitPath(lastTarball).tail.replace(".tar.gz", "")
thisUpstream.error = false
except:
thisUpstream.error = true
finally:
client.close()
return thisUpstream
proc makeChangelog*(name, version, mName, mMail: string): string =
#[
Example:
------------------------------------------------------------------
parrot-interface (4.10+parrot10) rolling-testing; urgency=medium
* Update debian standard version
* Update new copyright
* Fix invalid section name
-- Nong Hoang Tu <dmknght@parrotsec.org> Thu, 20 Aug 2020 01:18:06 +0700
====================================================================
Expected format:
------------------------------------------------------------------
<project name> (<version>-parrot0) rolling-testing; urgency=medium
* [Parrot Src Updater] Update to version <version>
-- <Maintainer_name> \<<Maintainer email>/> <Date: Thu, 20 Aug 2020> <Time: 01:18:06> <time_zone>
====================================================================
]#
result = name & " (" & version & "-parrot0) rolling-testing; urgency=medium\n\n"
result &= " * [Parrot Src Updater] Update to version " & version & "\n\n"
result &= " -- " & mName & " <" & mMail & "> " & getChangelogTime() & "\n\n"
proc writeChangelog*(path: string): bool =
#[
Switch to master branch and write changelog
]#
setCurrentDir(path)
let
cmdBranch = "git checkout master"
execBranch = execShellCmd(cmdBranch)
if execBranch != 0:
fullLog.log(lvlError, makeLogMsg(path, "Error while switching to master branch"))
return false
let
cmd = "gbp dch --distribution=rolling-testing"
execResult = execShellCmd(cmd)
if execResult != 0:
fullLog.log(lvlError, makeLogMsg(path, "Error while generating changelog by gbp dch"))
return false
# return true
# let cmdPush = "gbp push"
# try:
# let execPush = execShellCmd(cmdPush)
# if execPush != 0:
# fullLog.log(lvlError, makeLogMsg(path, "Error while pushing new version"))
# return false
# else:
# fullLog.log(lvlInfo, makeLogMsg(path, "Push latest commits completed"))
# except:
# fullLog.log(lvlError, makeLogMsg(path, getCurrentExceptionMsg()))
# return false
# return true
\ No newline at end of file
#[
This module is for get version of debian project
We use it for logging and create changelog
]#
import strutils
import parseutils
import httpClient
import regex
import core
import osproc
proc parseVersion(line: string): string =
#[
Return full version with subver from debian changelog
]#
var startAdding = false
for char in line:
if startAdding:
if char == ')':
return result
else:
result &= char
if char == '(':
startAdding = true
proc parseChangelogVersion*(line: string): string =
#[
Get version from debian/changelog
line: The line contains version number of project
Syntax: `anonsurf (3.1.7+parrot0) rolling-testing; urgency=medium`
We parses 3.1.7 only
https://github.com/rgeissert/devscripts/blob/master/scripts/uscan.pl#L543
Fix: Using offical regex method from uscan allow us to fix problems from projects
like mimikatz
NOTICE: for native projects like radare2, version will be 4.3.1+dfsg and
regex won't check it. However we are applying this code for non-native projects
so we can ignore this
]#
result = parseVersion(line)
let
subVer = findAndCaptureAll(result, re("-[^-]+$"))[0]
epoch = findAndCaptureAll(result, re("^\\d+:"))
if len(epoch) > 0:
if not isEmptyOrWhitespace(epoch[0]):
result = result.replace(epoch[0], "")
return result.replace(subVer, "")
proc localVersion*(path: string): string =
#[
path: string = The dirName(path): full project path of debian project
prjName: Name of current project that we check
We check if line starts with project name to make sure we don't miss any empty line
Example: This is syntax of latest debian changelog
```anonsurf (3.1.7+parrot0) rolling-testing; urgency=medium
* Fix some crash errors in GUI
* Fix Tor doesn't start at boot by adding Wants to sys-unit
-- Nong Hoang "DmKnght" Tu <dmknght@parrotsec.org> Fri, 11 Dec 2020 18:02:13 +0700
```
We make sure first line is always true by starting with only project name, then
return the parseChangelogVersion to get version number
Fixed mimikatz version error: https://github.com/rgeissert/devscripts/blob/master/scripts/uscan.pl#L543
Method: offical method from uscan
Function: parseChangelogVersion
String: 2.2.0-20200918-fix (Detected by uscan)
Full version: 2.2.0-20200918-fix-1
]#
let
fullPath = dirName(path) & "debian/changelog"
controlPath = dirName(path) & "debian/control"
var ctrlName = ""
for line in lines(controlPath):
if line.startsWith("Source: "):
ctrlName = line.split(" ")[1]
break
for line in lines(fullPath):
# Fixed can't get local version of gef (gdb-gef)
# solution: use source from control
if line.startsWith(ctrlName):
return parseChangelogVersion(line)
proc gitVersion(url, r_text: string): string =
#[
We use httpClient to access URL
(-d:ssl is required when compile code)
Then use regex to capture first match