Neels Hofmeyr | 0a1bdff | 2017-08-13 03:22:42 +0200 | [diff] [blame] | 1 | #!/usr/bin/env python3 |
| 2 | ''' |
| 3 | Generate a top-level makefile that builds the Osmocom 2G + 3G network components. |
| 4 | |
Neels Hofmeyr | 450dac7 | 2017-08-22 19:27:08 +0200 | [diff] [blame] | 5 | ./gen_makefile.py projects.deps [configure.opts [more.opts]] [-o Makefile.output] |
Neels Hofmeyr | 0a1bdff | 2017-08-13 03:22:42 +0200 | [diff] [blame] | 6 | |
| 7 | Configured by text files: |
| 8 | |
| 9 | *.deps: whitespace-separated listing of |
| 10 | project_name depends_on_project_1 depends_on_project_2 ... |
| 11 | |
| 12 | *.opts: whitespace-separated listing of |
| 13 | project_name --config-opt-1 --config-opt-2 ... |
| 14 | |
| 15 | Thus it is possible to choose between e.g. |
| 16 | - 2G+3G or 2G-only by picking a different projects_and_deps.conf, |
| 17 | - and between building each of those with or without mgcp transcoding support |
| 18 | by picking a different configure_opts.conf. |
| 19 | |
| 20 | From the Makefile nature, the dependencies extend, no need to repeat common deps. |
| 21 | |
| 22 | When this script is done, a Makefile has been generated that allows you to |
| 23 | build all projects at once by issuing 'make', but also to refresh only parts of |
| 24 | it when some bits in the middle have changed. The makefile keeps local progress |
| 25 | marker files like .make.libosmocore.configure; if such progress marker is |
| 26 | removed or becomes outdated, that step and all dependent ones are re-run. |
| 27 | This is helpful in daily hacking across several repositories. |
Neels Hofmeyr | 461c3bd | 2017-12-06 00:32:15 +0100 | [diff] [blame] | 28 | |
| 29 | Note that by default, this includes 'sudo ldconfig' calls following each |
| 30 | installation. You may want to permit your user to run 'sudo ldconfig' without |
| 31 | needing a password, e.g. by |
| 32 | |
| 33 | sudo sh -c "echo '$USER ALL= NOPASSWD: /sbin/ldconfig' > /etc/sudoers.d/${USER}_ldconfig" |
| 34 | |
| 35 | You can skip the 'sudo ldconfig' by issuing the --no-ldconfig option. |
| 36 | |
| 37 | You can run 'ldconfig' without sudo by issuing the --ldconfig-without-sudo option. |
| 38 | |
| 39 | By default, it is assumed that your user has write permission to /usr/local. If you |
| 40 | need sudo to install there, you may issue the --sudo-make-install option. |
Neels Hofmeyr | 2535a26 | 2018-01-16 16:34:32 +0100 | [diff] [blame] | 41 | |
| 42 | EXAMPLE: |
| 43 | |
| 44 | ./gen_makefile.py 3G+2G.deps default.opts iu.opts -I -m build |
| 45 | cd build |
| 46 | make |
| 47 | |
Neels Hofmeyr | 0a1bdff | 2017-08-13 03:22:42 +0200 | [diff] [blame] | 48 | ''' |
| 49 | |
| 50 | import sys |
| 51 | import os |
| 52 | import argparse |
| 53 | |
| 54 | parser = argparse.ArgumentParser(epilog=__doc__, formatter_class=argparse.RawTextHelpFormatter) |
| 55 | |
| 56 | parser.add_argument('projects_and_deps_file', |
| 57 | help='''Config file containing projects to build and |
| 58 | dependencies between those''') |
| 59 | |
Neels Hofmeyr | 450dac7 | 2017-08-22 19:27:08 +0200 | [diff] [blame] | 60 | parser.add_argument('configure_opts_files', |
Neels Hofmeyr | 0a1bdff | 2017-08-13 03:22:42 +0200 | [diff] [blame] | 61 | help='''Config file containing project name and |
| 62 | ./configure options''', |
Neels Hofmeyr | 450dac7 | 2017-08-22 19:27:08 +0200 | [diff] [blame] | 63 | nargs='*') |
Neels Hofmeyr | 0a1bdff | 2017-08-13 03:22:42 +0200 | [diff] [blame] | 64 | |
| 65 | parser.add_argument('-m', '--make-dir', dest='make_dir', |
| 66 | help='''Place Makefile in this dir (default: create |
| 67 | a new dir named after deps and opts files).''') |
| 68 | |
| 69 | parser.add_argument('-s', '--src-dir', dest='src_dir', default='./src', |
| 70 | help='Parent dir for all git clones.') |
| 71 | |
| 72 | parser.add_argument('-b', '--build-dir', dest='build_dir', |
| 73 | help='''Parent dir for all build trees (default: |
| 74 | directly in the make-dir).''') |
| 75 | |
Neels Hofmeyr | bffdc30 | 2017-12-06 00:31:49 +0100 | [diff] [blame] | 76 | parser.add_argument('-u', '--url', dest='url', default='git://git.osmocom.org', |
| 77 | help='''git clone base URL. Default is 'git://git.osmocom.org'. |
| 78 | e.g. with a config like this in your ~/.ssh/config: |
Neels Hofmeyr | 0a1bdff | 2017-08-13 03:22:42 +0200 | [diff] [blame] | 79 | host go |
| 80 | hostname gerrit.osmocom.org |
| 81 | port 29418 |
Neels Hofmeyr | bffdc30 | 2017-12-06 00:31:49 +0100 | [diff] [blame] | 82 | you may pass '-u ssh://go' to be able to submit to gerrit.''') |
Neels Hofmeyr | 0a1bdff | 2017-08-13 03:22:42 +0200 | [diff] [blame] | 83 | |
Neels Hofmeyr | 28d4be5 | 2018-03-16 03:44:07 +0100 | [diff] [blame] | 84 | parser.add_argument('-p', '--push-url', dest='push_url', default=None, |
| 85 | help='''git push-URL. Default is to not configure a separate push-URL.''') |
| 86 | |
Neels Hofmeyr | 0a1bdff | 2017-08-13 03:22:42 +0200 | [diff] [blame] | 87 | parser.add_argument('-o', '--output', dest='output', default='Makefile', |
| 88 | help='''Makefile filename (default: 'Makefile').''') |
| 89 | |
| 90 | parser.add_argument('-j', '--jobs', dest='jobs', default='9', |
| 91 | help='''-j option to pass to 'make'.''') |
| 92 | |
Neels Hofmeyr | 461c3bd | 2017-12-06 00:32:15 +0100 | [diff] [blame] | 93 | parser.add_argument('-I', '--sudo-make-install', dest='sudo_make_install', |
| 94 | action='store_true', |
| 95 | help='''run 'make install' step with 'sudo'.''') |
| 96 | |
| 97 | parser.add_argument('-L', '--no-ldconfig', dest='no_ldconfig', |
| 98 | action='store_true', |
| 99 | help='''omit the 'sudo ldconfig' step.''') |
| 100 | |
| 101 | parser.add_argument('--ldconfig-without-sudo', dest='ldconfig_without_sudo', |
| 102 | action='store_true', |
| 103 | help='''call just 'ldconfig', without sudo, which implies |
| 104 | root privileges (not recommended)''') |
| 105 | |
Neels Hofmeyr | 0a1bdff | 2017-08-13 03:22:42 +0200 | [diff] [blame] | 106 | args = parser.parse_args() |
| 107 | |
Neels Hofmeyr | 450dac7 | 2017-08-22 19:27:08 +0200 | [diff] [blame] | 108 | class listdict(dict): |
| 109 | 'a dict of lists { "a": [1, 2, 3], "b": [1, 2] }' |
| 110 | |
| 111 | def add(self, name, item): |
| 112 | l = self.get(name) |
| 113 | if not l: |
| 114 | l = [] |
| 115 | self[name] = l |
| 116 | l.append(item) |
| 117 | |
| 118 | def extend(self, name, l): |
| 119 | for v in l: |
| 120 | self.add(name, v) |
| 121 | |
| 122 | def add_dict(self, d): |
| 123 | for k,v in d.items(): |
| 124 | self.add(k, v) |
| 125 | |
| 126 | def extend_dict(self, d): |
| 127 | for k,v in d.items(): |
| 128 | l = self.extend(k, v) |
Neels Hofmeyr | 0a1bdff | 2017-08-13 03:22:42 +0200 | [diff] [blame] | 129 | |
| 130 | def read_projects_deps(path): |
| 131 | 'Read deps config and return tuples of (project_name, which-other-to-build-first).' |
| 132 | l = [] |
| 133 | for line in open(path): |
| 134 | line = line.strip() |
| 135 | if not line or line.startswith('#'): |
| 136 | continue |
| 137 | tokens = line.split() |
| 138 | l.append((tokens[0], tokens[1:])) |
| 139 | return l |
| 140 | |
| 141 | def read_configure_opts(path): |
| 142 | 'Read config opts file and return tuples of (project_name, config-opts).' |
| 143 | if not path: |
| 144 | return {} |
| 145 | return dict(read_projects_deps(path)) |
| 146 | |
Neels Hofmeyr | 28d4be5 | 2018-03-16 03:44:07 +0100 | [diff] [blame] | 147 | def gen_make(proj, deps, configure_opts, jobs, make_dir, src_dir, build_dir, url, push_url, sudo_make_install, no_ldconfig, ldconfig_without_sudo): |
Neels Hofmeyr | 0a1bdff | 2017-08-13 03:22:42 +0200 | [diff] [blame] | 148 | src_proj = os.path.join(src_dir, proj) |
| 149 | build_proj = os.path.join(build_dir, proj) |
| 150 | |
| 151 | make_to_src = os.path.relpath(src_dir, make_dir) |
| 152 | make_to_src_proj = os.path.relpath(src_proj, make_dir) |
| 153 | make_to_build_proj = os.path.relpath(build_proj, make_dir) |
| 154 | build_to_src = os.path.relpath(src_proj, build_proj) |
| 155 | |
| 156 | if configure_opts: |
| 157 | configure_opts_str = ' '.join(configure_opts) |
| 158 | else: |
| 159 | configure_opts_str = '' |
| 160 | |
| 161 | # special hack for libsmpp34: cannot build in parallel |
| 162 | if proj == 'libsmpp34': |
| 163 | jobs = 1 |
| 164 | |
| 165 | return r''' |
| 166 | ### {proj} ### |
| 167 | |
Neels Hofmeyr | 367cb97 | 2017-12-15 04:01:13 +0100 | [diff] [blame] | 168 | {proj}_configure_files := $(shell find {src_proj} -name "Makefile.am" -or -name "*.in" ) |
| 169 | {proj}_files := $(shell find {src_proj} -name "*.[hc]" -or -name "*.py" ) |
Neels Hofmeyr | 1c1e4d2 | 2017-09-11 01:32:50 +0200 | [diff] [blame] | 170 | |
Neels Hofmeyr | 0a1bdff | 2017-08-13 03:22:42 +0200 | [diff] [blame] | 171 | .make.{proj}.clone: |
| 172 | @echo "\n\n\n===== $@\n" |
| 173 | test -d {src} || mkdir -p {src} |
Neels Hofmeyr | 28d4be5 | 2018-03-16 03:44:07 +0100 | [diff] [blame] | 174 | test -d {src_proj} || ( git -C {src} clone "{url}/{proj}" "{proj}" && git -C "{src}/{proj}" remote set-url --push origin "{push_url}/{proj}" ) |
Neels Hofmeyr | 0a1bdff | 2017-08-13 03:22:42 +0200 | [diff] [blame] | 175 | touch $@ |
| 176 | |
| 177 | .make.{proj}.autoconf: .make.{proj}.clone {src_proj}/configure.ac |
| 178 | @echo "\n\n\n===== $@\n" |
Neels Hofmeyr | 7f586cf | 2017-11-14 09:42:57 +0100 | [diff] [blame] | 179 | -rm {src_proj}/.version |
Neels Hofmeyr | 0a1bdff | 2017-08-13 03:22:42 +0200 | [diff] [blame] | 180 | cd {src_proj}; autoreconf -fi |
| 181 | touch $@ |
| 182 | |
Neels Hofmeyr | 367cb97 | 2017-12-15 04:01:13 +0100 | [diff] [blame] | 183 | .make.{proj}.configure: .make.{proj}.autoconf {deps_installed} $({proj}_configure_files) |
Neels Hofmeyr | 0a1bdff | 2017-08-13 03:22:42 +0200 | [diff] [blame] | 184 | @echo "\n\n\n===== $@\n" |
| 185 | -chmod -R ug+w {build_proj} |
| 186 | -rm -rf {build_proj} |
| 187 | mkdir -p {build_proj} |
| 188 | cd {build_proj}; {build_to_src}/configure {configure_opts} |
| 189 | touch $@ |
| 190 | |
Neels Hofmeyr | 367cb97 | 2017-12-15 04:01:13 +0100 | [diff] [blame] | 191 | .make.{proj}.build: .make.{proj}.configure $({proj}_files) |
Neels Hofmeyr | 0a1bdff | 2017-08-13 03:22:42 +0200 | [diff] [blame] | 192 | @echo "\n\n\n===== $@\n" |
| 193 | $(MAKE) -C {build_proj} -j {jobs} check |
| 194 | touch $@ |
| 195 | |
| 196 | .make.{proj}.install: .make.{proj}.build |
| 197 | @echo "\n\n\n===== $@\n" |
Neels Hofmeyr | 461c3bd | 2017-12-06 00:32:15 +0100 | [diff] [blame] | 198 | {sudo_make_install}$(MAKE) -C {build_proj} install |
| 199 | {no_ldconfig}{sudo_ldconfig}ldconfig |
Neels Hofmeyr | 0a1bdff | 2017-08-13 03:22:42 +0200 | [diff] [blame] | 200 | touch $@ |
Neels Hofmeyr | 277f479 | 2017-08-29 12:30:32 +0200 | [diff] [blame] | 201 | |
Neels Hofmeyr | 3f93412 | 2017-08-29 12:31:59 +0200 | [diff] [blame] | 202 | {proj}: .make.{proj}.install |
| 203 | |
Neels Hofmeyr | 277f479 | 2017-08-29 12:30:32 +0200 | [diff] [blame] | 204 | .PHONY: {proj}-clean |
| 205 | {proj}-clean: |
| 206 | @echo "\n\n\n===== $@\n" |
| 207 | -chmod -R ug+w {build_proj} |
| 208 | -rm -rf {build_proj} |
| 209 | -rm -rf .make.{proj}.* |
Neels Hofmeyr | 0a1bdff | 2017-08-13 03:22:42 +0200 | [diff] [blame] | 210 | '''.format( |
| 211 | url=url, |
Neels Hofmeyr | 28d4be5 | 2018-03-16 03:44:07 +0100 | [diff] [blame] | 212 | push_url=push_url or url, |
Neels Hofmeyr | 0a1bdff | 2017-08-13 03:22:42 +0200 | [diff] [blame] | 213 | proj=proj, |
| 214 | jobs=jobs, |
| 215 | src=make_to_src, |
| 216 | src_proj=make_to_src_proj, |
| 217 | build_proj=make_to_build_proj, |
| 218 | build_to_src=build_to_src, |
| 219 | deps_installed=' '.join(['.make.%s.install' % d for d in deps]), |
Neels Hofmeyr | 461c3bd | 2017-12-06 00:32:15 +0100 | [diff] [blame] | 220 | configure_opts=configure_opts_str, |
| 221 | sudo_make_install='sudo ' if sudo_make_install else '', |
| 222 | no_ldconfig='#' if no_ldconfig else '', |
| 223 | sudo_ldconfig='' if ldconfig_without_sudo else 'sudo ' |
| 224 | ) |
Neels Hofmeyr | 0a1bdff | 2017-08-13 03:22:42 +0200 | [diff] [blame] | 225 | |
| 226 | |
| 227 | projects_deps = read_projects_deps(args.projects_and_deps_file) |
Neels Hofmeyr | 450dac7 | 2017-08-22 19:27:08 +0200 | [diff] [blame] | 228 | configure_opts = listdict() |
| 229 | configure_opts_files = sorted(args.configure_opts_files or []) |
| 230 | for configure_opts_file in configure_opts_files: |
| 231 | r = read_configure_opts(configure_opts_file) |
| 232 | configure_opts.extend_dict(read_configure_opts(configure_opts_file)) |
Neels Hofmeyr | 0a1bdff | 2017-08-13 03:22:42 +0200 | [diff] [blame] | 233 | |
| 234 | make_dir = args.make_dir |
| 235 | if not make_dir: |
Neels Hofmeyr | 450dac7 | 2017-08-22 19:27:08 +0200 | [diff] [blame] | 236 | deps_name = args.projects_and_deps_file.replace('.deps', '') |
| 237 | opts_names = '+'.join([f.replace('.opts', '') for f in configure_opts_files]) |
| 238 | make_dir = 'make-%s-%s' % (deps_name, opts_names) |
Neels Hofmeyr | 0a1bdff | 2017-08-13 03:22:42 +0200 | [diff] [blame] | 239 | |
| 240 | if not os.path.isdir(make_dir): |
| 241 | os.makedirs(make_dir) |
| 242 | |
| 243 | build_dir = args.build_dir |
| 244 | if not build_dir: |
| 245 | build_dir = make_dir |
| 246 | |
| 247 | output = os.path.join(make_dir, args.output) |
| 248 | print('Writing to %r' % output) |
| 249 | |
| 250 | with open(output, 'w') as out: |
| 251 | out.write('# This Makefile was generated by %s\n' % os.path.basename(sys.argv[0])) |
| 252 | |
| 253 | # convenience: add a regen target that updates the generated makefile itself |
| 254 | out.write(r''' |
| 255 | default: all |
| 256 | |
Neels Hofmeyr | 1b0d34f | 2018-03-16 03:46:08 +0100 | [diff] [blame^] | 257 | .PHONY: all_debug |
| 258 | all_debug: |
| 259 | $(MAKE) --dry-run -d all | grep "is newer than target" |
| 260 | $(MAKE) all |
| 261 | |
Neels Hofmeyr | 0a1bdff | 2017-08-13 03:22:42 +0200 | [diff] [blame] | 262 | # regenerate this Makefile, in case the deps or opts changed |
| 263 | .PHONY: regen |
| 264 | regen: |
Neels Hofmeyr | 28d4be5 | 2018-03-16 03:44:07 +0100 | [diff] [blame] | 265 | {script} {projects_and_deps} {configure_opts} -m {make_dir} -o {makefile} -s {src_dir} -b {build_dir} -u "{url}" -p "{push_url}"{sudo_make_install}{no_ldconfig}{ldconfig_without_sudo} |
Neels Hofmeyr | 0a1bdff | 2017-08-13 03:22:42 +0200 | [diff] [blame] | 266 | |
| 267 | '''.format( |
| 268 | script=os.path.relpath(sys.argv[0], make_dir), |
| 269 | projects_and_deps=os.path.relpath(args.projects_and_deps_file, make_dir), |
Neels Hofmeyr | 450dac7 | 2017-08-22 19:27:08 +0200 | [diff] [blame] | 270 | configure_opts=' '.join([os.path.relpath(f, make_dir) for f in configure_opts_files]), |
Neels Hofmeyr | 0a1bdff | 2017-08-13 03:22:42 +0200 | [diff] [blame] | 271 | make_dir='.', |
| 272 | makefile=args.output, |
| 273 | src_dir=os.path.relpath(args.src_dir, make_dir), |
| 274 | build_dir=os.path.relpath(build_dir, make_dir), |
Neels Hofmeyr | 461c3bd | 2017-12-06 00:32:15 +0100 | [diff] [blame] | 275 | url=args.url, |
Neels Hofmeyr | 28d4be5 | 2018-03-16 03:44:07 +0100 | [diff] [blame] | 276 | push_url=args.push_url, |
Neels Hofmeyr | 461c3bd | 2017-12-06 00:32:15 +0100 | [diff] [blame] | 277 | sudo_make_install=' -I' if args.sudo_make_install else '', |
| 278 | no_ldconfig=' -L' if args.no_ldconfig else '', |
| 279 | ldconfig_without_sudo=' --ldconfig-without-sudo' if args.ldconfig_without_sudo else '' |
Neels Hofmeyr | 0a1bdff | 2017-08-13 03:22:42 +0200 | [diff] [blame] | 280 | )) |
| 281 | |
Neels Hofmeyr | e274d35 | 2017-08-22 17:31:03 +0200 | [diff] [blame] | 282 | # convenience target: clone all repositories first |
| 283 | out.write('clone: \\\n\t' + ' \\\n\t'.join([ '.make.%s.clone' % p for p, d in projects_deps ]) + '\n\n') |
| 284 | |
Neels Hofmeyr | 277f479 | 2017-08-29 12:30:32 +0200 | [diff] [blame] | 285 | # convenience target: clean all |
| 286 | out.write('clean: \\\n\t' + ' \\\n\t'.join([ '%s-clean' % p for p, d in projects_deps ]) + '\n\n') |
| 287 | |
Neels Hofmeyr | 0a1bdff | 2017-08-13 03:22:42 +0200 | [diff] [blame] | 288 | # now the actual useful build rules |
Neels Hofmeyr | 1c1e4d2 | 2017-09-11 01:32:50 +0200 | [diff] [blame] | 289 | out.write('all: clone all-install\n\n') |
| 290 | |
| 291 | out.write('all-install: \\\n\t' + ' \\\n\t'.join([ '.make.%s.install' % p for p, d in projects_deps ]) + '\n\n') |
Neels Hofmeyr | 0a1bdff | 2017-08-13 03:22:42 +0200 | [diff] [blame] | 292 | |
| 293 | for proj, deps in projects_deps: |
| 294 | out.write(gen_make(proj, deps, configure_opts.get(proj), args.jobs, |
Neels Hofmeyr | 28d4be5 | 2018-03-16 03:44:07 +0100 | [diff] [blame] | 295 | make_dir, args.src_dir, build_dir, args.url, args.push_url, |
Neels Hofmeyr | 461c3bd | 2017-12-06 00:32:15 +0100 | [diff] [blame] | 296 | args.sudo_make_install, args.no_ldconfig, |
| 297 | args.ldconfig_without_sudo)) |
Neels Hofmeyr | d4d8848 | 2017-08-22 19:27:28 +0200 | [diff] [blame] | 298 | |
| 299 | # vim: expandtab tabstop=2 shiftwidth=2 |